edu-rdm-integration 3.9.1__py3-none-any.whl → 3.10.0__py3-none-any.whl

edu_rdm_integration/{base.py → pipelines/transfer/mixins.py}

@@ -1,74 +1,48 @@
-import logging
-from abc import (
-    ABC,
-    abstractmethod,
-)
 from collections import (
     defaultdict,
 )
 from typing import (
-    Optional,
-)
-
-from django.db.models import (
-    Q,
+    TYPE_CHECKING,
 )
 
 from m3_db_utils.consts import (
     DEFAULT_ORDER_NUMBER,
 )
 
-from edu_rdm_integration.consts import (
+from edu_rdm_integration.core.consts import (
     REGIONAL_DATA_MART_INTEGRATION_COLLECTING_DATA,
     REGIONAL_DATA_MART_INTEGRATION_EXPORTING_DATA,
 )
-from edu_rdm_integration.helpers import (
-    UploadStatusHelper,
-    get_collecting_managers_max_period_ended_dates,
-    get_exporting_managers_max_period_ended_dates,
-    save_command_log_link,
+from edu_rdm_integration.core.storages import (
+    RegionalDataMartEntityStorage,
 )
-from edu_rdm_integration.models import (
-    ExportingDataSubStageUploaderClientLog,
+from edu_rdm_integration.rdm_entities.models import (
     RegionalDataMartEntityEnum,
-    TransferredEntity,
 )
-from edu_rdm_integration.storages import (
-    RegionalDataMartEntityStorage,
+from edu_rdm_integration.stages.collect_data.helpers import (
+    get_collecting_managers_max_period_ended_dates,
+)
+from edu_rdm_integration.stages.export_data.helpers import (
+    get_exporting_managers_max_period_ended_dates,
 )
 
 
-class BaseOperationData(ABC):
-    """Базовый класс операций с данными."""
-
-    def __init__(self, **kwargs):
-        # Идентификатор команды для передачи сигналу manager_created
-        self.command_id: Optional[int] = kwargs.get('command_id')
-
-        self._file_handler: Optional[logging.FileHandler] = None
-
-        self._add_file_handler()
-
-    @property
-    @abstractmethod
-    def _log_file_path(self) -> str:
-        """Путь до лог файла."""
-
-    def _add_file_handler(self) -> None:
-        """Добавляет обработчик логов."""
-        if self.command_id:
-            self._file_handler = logging.FileHandler(self._log_file_path)
+if TYPE_CHECKING:
+    from datetime import (
+        datetime,
+    )
 
-            logging.getLogger('info_logger').addHandler(self._file_handler)
-            logging.getLogger('exception_logger').addHandler(self._file_handler)
+    from django.db.models import (
+        QuerySet,
+    )
 
-    def _remove_file_handler(self) -> None:
-        """Удаляет обработчик логов."""
-        if self._file_handler:
-            logging.getLogger('info_logger').removeHandler(self._file_handler)
-            logging.getLogger('exception_logger').removeHandler(self._file_handler)
+    from function_tools.managers import (
+        RunnerManager,
+    )
 
-            self._file_handler.close()
+    from edu_rdm_integration.pipelines.transfer.models import (
+        TransferredEntity,
+    )
 
 
 class BaseTransferLatestEntitiesDataMixin:
@@ -87,12 +61,11 @@ class BaseTransferLatestEntitiesDataMixin:
         self._entites_models_map = defaultdict(list)
 
     def get_entity_qs(self) -> 'QuerySet[TransferredEntity]':
-        """Возвращает сущностей сбора и выгрузки."""
+        """Возвращает QuerySet сущностей сбора и выгрузки."""
         raise NotImplementedError
 
     def _collect_transferred_entities(self) -> None:
         """Собирает сущности РВД, по которым будет произведен сбор и экспорт данных."""
-
         self._transferred_entities = [
             (RegionalDataMartEntityEnum.get_model_enum_value(key=entity), export_enabled)
             for entity, export_enabled in self.get_entity_qs().values_list('entity', 'export_enabled')
@@ -101,8 +74,11 @@ class BaseTransferLatestEntitiesDataMixin:
         # Собираем словарь по сущностям с моделями для сборки
         for entity, _ in self._transferred_entities:
             self._entites_models_map[entity.key].extend(
-                (model_enum for model_enum in (*entity.additional_model_enums, entity.main_model_enum)
-                 if model_enum.order_number != DEFAULT_ORDER_NUMBER)
+                (
+                    model_enum
+                    for model_enum in (*entity.additional_model_enums, entity.main_model_enum)
+                    if model_enum.order_number != DEFAULT_ORDER_NUMBER
+                )
             )
 
     def _collect_managers(self) -> None:

edu_rdm_integration/pipelines/transfer/models.py

@@ -0,0 +1,48 @@
+from django.db.models import (
+    CASCADE,
+    BooleanField,
+    OneToOneField,
+    PositiveIntegerField,
+)
+from m3 import (
+    json_encode,
+)
+from m3.db import (
+    BaseObjectModel,
+)
+
+from edu_rdm_integration.pipelines.transfer.enums import (
+    EntityLevelQueueTypeEnum,
+)
+from edu_rdm_integration.rdm_entities.models import (
+    RegionalDataMartEntityEnum,
+)
+
+
+class TransferredEntity(BaseObjectModel):
+    """Сущность, по которой должен быть произведен сбор и экспорт данных."""
+
+    entity = OneToOneField(
+        to=RegionalDataMartEntityEnum,
+        verbose_name='Сущность',
+        on_delete=CASCADE,
+    )
+    export_enabled = BooleanField(
+        verbose_name='Включение экспорта для сущности',
+        default=True,
+    )
+    queue_level = PositiveIntegerField(
+        choices=EntityLevelQueueTypeEnum.get_choices(),
+        default=EntityLevelQueueTypeEnum.BASE,
+        verbose_name='Уровень очереди',
+    )
+
+    class Meta:
+        db_table = 'rdm_transferred_entity'
+        verbose_name = 'Сущность, по которой должен быть произведен сбор и экспорт данных'
+        verbose_name_plural = 'Сущности, по которым должен быть произведен сбор и экспорт данных'
+
+    @json_encode
+    def no_export(self):
+        """Формирует отображение признака отключения экспорта."""
+        return 'Нет' if self.export_enabled else 'Да'

edu_rdm_integration/{tasks.py → pipelines/transfer/tasks.py}

@@ -1,5 +1,5 @@
-from collections import (
-    defaultdict,
+from datetime import (
+    datetime,
 )
 from typing import (
     TYPE_CHECKING,
@@ -13,13 +13,13 @@ from celery.schedules import (
 from django.conf import (
     settings,
 )
-from django.core.cache import (
-    cache,
-)
 from django.utils import (
     timezone,
 )
 
+from educommon import (
+    logger,
+)
 from educommon.async_task.models import (
     AsyncTaskType,
     RunningTask,
@@ -27,122 +27,52 @@ from educommon.async_task.models import (
 from educommon.async_task.tasks import (
     UniquePeriodicAsyncTask,
 )
+from educommon.django.db.mixins.validation import (
+    QuerySet,
+)
 from educommon.utils.date import (
     get_today_min_datetime,
 )
 
-from edu_rdm_integration.base import (
-    BaseTransferLatestEntitiesDataMixin,
-)
-from edu_rdm_integration.collect_and_export_data.models import (
-    EduRdmCollectDataCommandProgress,
-    EduRdmExportDataCommandProgress,
-)
-from edu_rdm_integration.collect_data.collect import (
-    BaseCollectLatestModelsData,
-)
-from edu_rdm_integration.collect_data.helpers import (
-    set_failed_status_suspended_collecting_data_stages,
-)
-from edu_rdm_integration.consts import (
+from edu_rdm_integration.core.consts import (
     FAST_TRANSFER_TASK_QUEUE_NAME,
     LONG_TRANSFER_TASK_QUEUE_NAME,
     TASK_QUEUE_NAME,
 )
-from edu_rdm_integration.enums import (
+from edu_rdm_integration.core.enums import (
     CommandType,
+)
+from edu_rdm_integration.core.helpers import (
+    save_command_log_link,
+)
+from edu_rdm_integration.pipelines.transfer.enums import (
     EntityLevelQueueTypeEnum,
-    FileUploadStatusEnum,
 )
-from edu_rdm_integration.export_data.export import (
-    ExportLatestEntitiesData,
-    UploadData,
+from edu_rdm_integration.pipelines.transfer.mixins import (
+    BaseTransferLatestEntitiesDataMixin,
 )
-from edu_rdm_integration.export_data.helpers import (
-    set_failed_status_suspended_exporting_data_stages,
+from edu_rdm_integration.pipelines.transfer.models import (
+    TransferredEntity,
 )
-from edu_rdm_integration.export_data.queue import (
-    RdmRedisSubStageAttachmentQueue,
+from edu_rdm_integration.stages.collect_data.models import (
+    EduRdmCollectDataCommandProgress,
 )
-from edu_rdm_integration.helpers import (
-    UploadStatusHelper,
-    save_command_log_link,
+from edu_rdm_integration.stages.collect_data.operations import (
+    BaseCollectLatestModelsData,
 )
-from edu_rdm_integration.models import (
-    ExportingDataSubStageUploaderClientLog,
-    TransferredEntity,
+from edu_rdm_integration.stages.export_data.models import (
+    EduRdmExportDataCommandProgress,
+)
+from edu_rdm_integration.stages.export_data.operations import (
+    ExportLatestEntitiesData,
 )
 
 
 if TYPE_CHECKING:
-    from datetime import (
-        datetime,
-    )
-
-
-class RDMCheckUploadStatus(UniquePeriodicAsyncTask):
-    """Периодическая задача для сбора статусов по загрузке файла в витрину."""
-
-    queue = TASK_QUEUE_NAME
-    routing_key = TASK_QUEUE_NAME
-    description = 'Сбор статусов загрузки данных в витрину "Региональная витрина данных"'
-    lock_expire_seconds = settings.RDM_UPLOAD_STATUS_TASK_LOCK_EXPIRE_SECONDS
-    task_type = AsyncTaskType.UNKNOWN
-    run_every = crontab(
-        minute=settings.RDM_UPLOAD_STATUS_TASK_MINUTE,
-        hour=settings.RDM_UPLOAD_STATUS_TASK_HOUR,
-        day_of_week=settings.RDM_UPLOAD_STATUS_TASK_DAY_OF_WEEK,
-    )
-
-    def process(self, *args, **kwargs):
-        """Выполнение."""
-        super().process(*args, **kwargs)
-
-        # Получаем незавершенные загрузки данных в витрину
-        in_progress_uploads = ExportingDataSubStageUploaderClientLog.objects.filter(
-            file_upload_status=FileUploadStatusEnum.IN_PROGRESS,
-            is_emulation=False,
-        ).select_related('attachment')
-
-        UploadStatusHelper(in_progress_uploads, cache).run()
-
-
-class CheckSuspendedExportedStagePeriodicTask(UniquePeriodicAsyncTask):
-    """Периодическая задача поиска зависших этапов/подэтапов экспорта."""
-
-    queue = TASK_QUEUE_NAME
-    routing_key = TASK_QUEUE_NAME
-    description = 'Поиск зависших этапов/подэтапов экспорта в "Региональная витрина данных"'
-    lock_expire_seconds = settings.RDM_CHECK_SUSPEND_TASK_LOCK_EXPIRE_SECONDS
-    task_type = AsyncTaskType.SYSTEM
-    run_every = crontab(
-        minute=settings.RDM_CHECK_SUSPEND_TASK_MINUTE,
-        hour=settings.RDM_CHECK_SUSPEND_TASK_HOUR,
-        day_of_week=settings.RDM_CHECK_SUSPEND_TASK_DAY_OF_WEEK,
+    from django.db.models import (
+        QuerySet,
     )
 
-    def process(self, *args, **kwargs):
-        """Выполнение задачи."""
-        super().process(*args, **kwargs)
-
-        change_status_collecting_result = set_failed_status_suspended_collecting_data_stages()
-        change_status_exporting_result = set_failed_status_suspended_exporting_data_stages()
-
-        task_result = {
-            'Прервано сборок': (
-                f'Этапов {change_status_collecting_result["change_stage_count"]}'
-                f' и подэтапов {change_status_collecting_result["change_sub_stage_count"]}'
-            ),
-            'Прервано выгрузок': (
-                f'Этапов {change_status_exporting_result["change_stage_count"]}'
-                f' и подэтапов {change_status_exporting_result["change_sub_stage_count"]}'
-            ),
-        }
-
-        self.set_progress(
-            values=task_result
-        )
-
 
 class BaseTransferLatestEntitiesDataPeriodicTask(BaseTransferLatestEntitiesDataMixin, UniquePeriodicAsyncTask):
     """Базовая периодическая задача сбора и выгрузки данных для переиспользования в разных очередях."""
@@ -186,8 +116,7 @@ class BaseTransferLatestEntitiesDataPeriodicTask(BaseTransferLatestEntitiesDataM
         """Создает команду сбора данных моделей РВД."""
         manager = self._collecting_data_managers[model]
         manager_last_collected = (
-            self._collecting_data_manager_to_logs_period_end.get(manager.uuid)
-            or get_today_min_datetime()
+            self._collecting_data_manager_to_logs_period_end.get(manager.uuid) or get_today_min_datetime()
         )
 
         period_started_at = manager_last_collected
@@ -218,9 +147,10 @@ class BaseTransferLatestEntitiesDataPeriodicTask(BaseTransferLatestEntitiesDataM
                 type=CommandType.AUTO,
             )
 
+        return None
+
     def _prepare_collect_model_data_class(
-        self,
-        command: EduRdmCollectDataCommandProgress
+        self, command: EduRdmCollectDataCommandProgress
     ) -> BaseCollectLatestModelsData:
         """Подготавливает объект класса сбора данных моделей РВД."""
         return BaseCollectLatestModelsData(
@@ -252,9 +182,13 @@ class BaseTransferLatestEntitiesDataPeriodicTask(BaseTransferLatestEntitiesDataM
         self._period_ended_at = None
         self.prepare_collect_export_managers()
 
-        task_id = RunningTask.objects.filter(
-            pk=self.request.id,
-        ).values_list('pk', flat=True).first()
+        task_id = (
+            RunningTask.objects.filter(
+                pk=self.request.id,
+            )
+            .values_list('pk', flat=True)
+            .first()
+        )
 
         collected_entity_models = set()
 
@@ -267,13 +201,17 @@ class BaseTransferLatestEntitiesDataPeriodicTask(BaseTransferLatestEntitiesDataM
                     collected_entity_models.add(model_enum_value.key)
                     try:
                         self._run_collect_model_data(model_enum_value.key, task_id)
-                    except Exception:
+                    except Exception as e:
+                        logger.warning(e)
+
                         continue
 
             try:
                 if export_enabled:
                     self._run_export_entity_data(entity_enum.key, task_id)
-            except Exception:
+            except Exception as e:
+                logger.warning(e)
+
                 continue
 
 
@@ -292,6 +230,7 @@ class TransferLatestEntitiesDataPeriodicTask(BaseTransferLatestEntitiesDataPerio
     )
 
     def get_entity_qs(self) -> 'QuerySet[TransferredEntity]':
+        """Возвращает QuerySet сущностей сбора и выгрузки."""
         return TransferredEntity.objects.filter(queue_level=EntityLevelQueueTypeEnum.BASE)
 
 
@@ -310,6 +249,7 @@ class TransferLatestEntitiesDataFastPeriodicTask(BaseTransferLatestEntitiesDataP
     )
 
     def get_entity_qs(self) -> 'QuerySet[TransferredEntity]':
+        """Возвращает QuerySet сущностей сбора и выгрузки."""
         return TransferredEntity.objects.filter(queue_level=EntityLevelQueueTypeEnum.FAST)
 
 
@@ -328,50 +268,11 @@ class TransferLatestEntitiesDataLongPeriodicTask(BaseTransferLatestEntitiesDataP
     )
 
     def get_entity_qs(self) -> 'QuerySet[TransferredEntity]':
+        """Возвращает QuerySet сущностей сбора и выгрузки."""
         return TransferredEntity.objects.filter(queue_level=EntityLevelQueueTypeEnum.LONG)
 
 
-class UploadDataAsyncTask(UniquePeriodicAsyncTask):
-    """Формирование очереди файлов и их отправка."""
-
-    queue = TASK_QUEUE_NAME
-    routing_key = TASK_QUEUE_NAME
-    description = 'Отправка данных в витрину "Региональная витрина данных"'
-    lock_expire_seconds = settings.RDM_UPLOAD_DATA_TASK_LOCK_EXPIRE_SECONDS
-    task_type = AsyncTaskType.SYSTEM
-    run_every = crontab(
-        minute=settings.RDM_UPLOAD_DATA_TASK_MINUTE,
-        hour=settings.RDM_UPLOAD_DATA_TASK_HOUR,
-        day_of_week=settings.RDM_UPLOAD_DATA_TASK_DAY_OF_WEEK,
-    )
-
-    def process(self, *args, **kwargs):
-        """Выполнение."""
-        super().process(*args, **kwargs)
-
-        queue = RdmRedisSubStageAttachmentQueue()
-        upload_data = UploadData(
-            data_cache=cache,
-            queue=queue,
-        )
-
-        upload_result = upload_data.upload_data()
-
-        task_result = {
-            'Общий объем отправленных файлов': f"{upload_result['total_file_size']}",
-            'Очередь отправки переполнена': 'Да' if upload_result['queue_is_full'] else 'Нет',
-            'Сущности, отправленные в витрину': upload_result['uploaded_entities']
-        }
-
-        self.set_progress(
-            values=task_result
-        )
-
-
 celery_app = celery.app.app_or_default()
-celery_app.register_task(RDMCheckUploadStatus)
-celery_app.register_task(CheckSuspendedExportedStagePeriodicTask)
 celery_app.register_task(TransferLatestEntitiesDataPeriodicTask)
-celery_app.register_task(UploadDataAsyncTask)
 celery_app.register_task(TransferLatestEntitiesDataFastPeriodicTask)
-celery_app.register_task(TransferLatestEntitiesDataLongPeriodicTask)
+celery_app.register_task(TransferLatestEntitiesDataLongPeriodicTask)

edu_rdm_integration/{registry → pipelines/transfer}/ui.py

@@ -13,7 +13,7 @@ from educommon.utils.ui import (
     append_template_globals,
 )
 
-from edu_rdm_integration.models import (
+from edu_rdm_integration.pipelines.transfer.models import (
     TransferredEntity,
 )
 
@@ -25,12 +25,8 @@ class TransferredEntityListWindow(BaseListWindow):
         """Инициализация компонентов окна."""
         super()._init_components()
 
-        self.export_off_button = ExtButton(
-            text='Отключить экспорт', handler='offExport'
-        )
-        self.export_on_button = ExtButton(
-            text='Включить экспорт', handler='onExport'
-        )
+        self.export_off_button = ExtButton(text='Отключить экспорт', handler='offExport')
+        self.export_on_button = ExtButton(text='Включить экспорт', handler='onExport')
 
     def _do_layout(self):
         """Размещение компонентов окна на форме."""
@@ -43,9 +39,7 @@ class TransferredEntityListWindow(BaseListWindow):
         super().set_params(params, *args, **kwargs)
 
         append_template_globals(self, 'ui-js/transferred-entity-list.js')
-        self.export_change_action_url = (
-            params['pack'].export_change_action.get_absolute_url()
-        )
+        self.export_change_action_url = params['pack'].export_change_action.get_absolute_url()
         self.pack = params['pack']
 
 

edu_rdm_integration/rdm_entities/apps.py

@@ -0,0 +1,11 @@
+from django.apps import (
+    AppConfig,
+)
+
+
+class RDMEntitiesAppConfig(AppConfig):
+    """Приложение для работы с сущностями РВД."""
+
+    name = 'edu_rdm_integration.rdm_entities'
+    label = 'edu_rdm_integration_entities'
+    verbose_name = 'Сущности РВД'