edu-rdm-integration 3.3.6__py3-none-any.whl → 3.4.0__py3-none-any.whl

edu_rdm_integration/collect_data/tests.py

@@ -32,6 +32,9 @@ from educommon.audit_log.models import (
 from educommon.integration_entities.enums import (
     EntityLogOperation,
 )
+from educommon.utils.phone_number.phone_number import (
+    PhoneNumber,
+)
 from function_tools.models import (
     Entity,
     EntityType,
@@ -166,6 +169,8 @@ class BaseCollectingFunctionTestCase(TestCase):
                 fields[key] = value.name
             elif isinstance(value, list):
                 fields[key] = f'{{{",".join(map(str, value))}}}'
+            elif isinstance(value, PhoneNumber):
+                fields[key] = value.cleaned
             else:
                 fields[key] = value
 

edu_rdm_integration/enum_register/mixins.py

@@ -1,7 +1,9 @@
 from typing import (
     Any,
     Dict,
+    NamedTuple,
     Optional,
+    Type,
     Tuple,
 )
 
@@ -23,6 +25,9 @@ from edu_rdm_integration.models import (
     RegionalDataMartEntityEnum,
     RegionalDataMartModelEnum,
 )
+from edu_rdm_integration.typing import (
+    MODEL_TYPE_VAR,
+)
 from edu_rdm_integration.utils import (
     camel_to_underscore,
 )
@@ -108,3 +113,17 @@ class ModelEnumRegisterMixin(BaseEnumRegisterMixin):
         register_params['loggable_models'] = cls.loggable_models
 
         return register_params
+
+
+class FromNamedTupleMixin:
+    """Миксин получения экземпляра модели из получаемого кэша значений."""
+
+    @classmethod
+    def from_namedtuple(cls: Type[MODEL_TYPE_VAR], namedtuple: NamedTuple) -> MODEL_TYPE_VAR:
+        """Создает экземпляр класса из NamedTuple."""
+        return cls(
+            **{
+                field: getattr(namedtuple, field)
+                for field in [f.column for f in cls._meta.get_fields()]
+            }
+        )

edu_rdm_integration/export_data/export.py

@@ -15,6 +15,7 @@ from typing import (
     List,
     Set,
     Type,
+    Union,
 )
 
 from django.conf import (
@@ -69,6 +70,13 @@ from edu_rdm_integration.consts import (
 from edu_rdm_integration.enums import (
     FileUploadStatusEnum,
 )
+from edu_rdm_integration.export_data.export_manger import (
+    ExportEntityQueueSender,
+    WorkerSender,
+)
+from edu_rdm_integration.export_data.queue import (
+    Queue,
+)
 from edu_rdm_integration.helpers import (
     get_exporting_managers_max_period_ended_dates,
 )
@@ -77,6 +85,9 @@ from edu_rdm_integration.models import (
     ExportingDataSubStageStatus,
     RegionalDataMartEntityEnum,
 )
+from edu_rdm_integration.redis_cache import (
+    AbstractCache,
+)
 from edu_rdm_integration.signals import (
     manager_created,
 )
@@ -121,8 +132,6 @@ class BaseExportEntitiesData(BaseOperationData):
         # Карта соответствия manager_id сущности и его основной модели
         self.manager_main_model_map: Dict[str, ModelBase] = {}
 
-        self._configure_agent_client()
-
     @property
     def _log_file_path(self) -> str:
         """
@@ -130,57 +139,6 @@ class BaseExportEntitiesData(BaseOperationData):
         """
         return os.path.join(settings.MEDIA_ROOT, settings.RDM_EXPORT_LOG_DIR, f'{self.command_id}.log')
 
-    def _configure_agent_client(self):
-        """
-        Конфигурирование клиента загрузчика данных в Витрину.
-
-        #TODO Вынужденная мера, т.к. при запуске команды не производится проверка готовности конфигов приложений.
-          # Нужно переработать механизм конфигурирования клиента загрузчика.
-        """
-        import uploader_client
-        from django.core.cache import (
-            DEFAULT_CACHE_ALIAS,
-            caches,
-        )
-        from uploader_client.contrib.rdm.interfaces.configurations import (
-            RegionalDataMartUploaderConfig,
-        )
-        if settings.RDM_UPLOADER_CLIENT_ENABLE_REQUEST_EMULATION:
-            uploader_client.set_config(
-                RegionalDataMartUploaderConfig(
-                    interface='uploader_client.contrib.rdm.interfaces.rest.OpenAPIInterfaceEmulation',
-                    url=settings.RDM_UPLOADER_CLIENT_URL,
-                    datamart_name=settings.RDM_UPLOADER_CLIENT_DATAMART_NAME,
-                    timeout=1,
-                    request_retries=1,
-                )
-            )
-        elif settings.RDM_UPLOADER_CLIENT_USE_PROXY_API:
-            uploader_client.set_config(
-                RegionalDataMartUploaderConfig(
-                    interface='uploader_client.contrib.rdm.interfaces.rest.ProxyAPIInterface',
-                    cache=caches[DEFAULT_CACHE_ALIAS],
-                    url=settings.RDM_UPLOADER_CLIENT_URL,
-                    datamart_name=settings.RDM_UPLOADER_CLIENT_DATAMART_NAME,
-                    timeout=settings.RDM_UPLOADER_CLIENT_REQUEST_TIMEOUT,
-                    request_retries=settings.RDM_UPLOADER_CLIENT_REQUEST_RETRIES,
-                    organization_ogrn=settings.RDM_UPLOADER_CLIENT_ORGANIZATION_OGRN,
-                    installation_name=settings.RDM_UPLOADER_CLIENT_INSTALLATION_NAME,
-                    installation_id=settings.RDM_UPLOADER_CLIENT_INSTALLATION_ID,
-                    username=settings.RDM_UPLOADER_CLIENT_USERNAME,
-                    password=settings.RDM_UPLOADER_CLIENT_PASSWORD,
-                )
-            )
-        else:
-            uploader_client.set_config(
-                RegionalDataMartUploaderConfig(
-                    url=settings.RDM_UPLOADER_CLIENT_URL,
-                    datamart_name=settings.RDM_UPLOADER_CLIENT_DATAMART_NAME,
-                    timeout=settings.RDM_UPLOADER_CLIENT_REQUEST_TIMEOUT,
-                    request_retries=settings.RDM_UPLOADER_CLIENT_REQUEST_RETRIES,
-                )
-            )
-
     def _has_stage_created_or_in_progress(self, manager_id: str, entity: str) -> bool:
         """Проверяет есть ли готовый к работе stage или в работе для данной сущности."""
         stage_created_or_in_progress = ExportingDataStage.objects.filter(
@@ -414,3 +372,105 @@ class ExportLatestEntitiesData(BaseExportLatestEntitiesData):
     def _get_async_task(self) -> Model:
         """Возвращает модель асинхронной задачи."""
         return RunningTask
+
+
+class UploadEntitiesData(BaseOperationData):
+    """Класс отправки файлов с сущностями в витрину."""
+
+    def __init__(
+        self,
+        entities: Iterable[str],
+        data_cache: AbstractCache,
+        queue: Queue,
+        **kwargs,
+    ):
+        super().__init__(**kwargs)
+
+        # Если сущности не указаны, берется значение по умолчанию - все сущности:
+        self.entities = entities if entities else RegionalDataMartEntityEnum.get_enum_data().keys()
+
+        self.data_cache = data_cache
+        self.queue = queue
+
+        self.task_id = kwargs.get('task_id')
+
+        self._configure_agent_client()
+
+    @property
+    def _log_file_path(self) -> Union[str, bytes]:
+        """
+        Путь до лог файла.
+        """
+        if self.command_id:
+            log_file_path = os.path.join(settings.MEDIA_ROOT, settings.RDM_UPLOAD_LOG_DIR, f'{self.command_id}.log')
+        else:
+            log_file_path = os.path.join(settings.MEDIA_ROOT, settings.RDM_UPLOAD_LOG_DIR, 'upload_entity.log')
+
+        return log_file_path
+
+    # TODO https://jira.bars.group/browse/EDUSCHL-22492. Вынужденная мера, т.к. при запуске команды не производится
+    #  проверка готовности конфигов приложений. Нужно переработать механизм конфигурирования клиента загрузчика.
+    def _configure_agent_client(self):
+        """
+        Конфигурирование клиента загрузчика данных в Витрину.
+        """
+        import uploader_client
+        from django.core.cache import (
+            DEFAULT_CACHE_ALIAS,
+            caches,
+        )
+        from uploader_client.contrib.rdm.interfaces.configurations import (
+            RegionalDataMartUploaderConfig,
+        )
+        if settings.RDM_UPLOADER_CLIENT_ENABLE_REQUEST_EMULATION:
+            uploader_client.set_config(
+                RegionalDataMartUploaderConfig(
+                    interface='uploader_client.contrib.rdm.interfaces.rest.OpenAPIInterfaceEmulation',
+                    url=settings.RDM_UPLOADER_CLIENT_URL,
+                    datamart_name=settings.RDM_UPLOADER_CLIENT_DATAMART_NAME,
+                    timeout=1,
+                    request_retries=1,
+                )
+            )
+        elif settings.RDM_UPLOADER_CLIENT_USE_PROXY_API:
+            uploader_client.set_config(
+                RegionalDataMartUploaderConfig(
+                    interface='uploader_client.contrib.rdm.interfaces.rest.ProxyAPIInterface',
+                    cache=caches[DEFAULT_CACHE_ALIAS],
+                    url=settings.RDM_UPLOADER_CLIENT_URL,
+                    datamart_name=settings.RDM_UPLOADER_CLIENT_DATAMART_NAME,
+                    timeout=settings.RDM_UPLOADER_CLIENT_REQUEST_TIMEOUT,
+                    request_retries=settings.RDM_UPLOADER_CLIENT_REQUEST_RETRIES,
+                    organization_ogrn=settings.RDM_UPLOADER_CLIENT_ORGANIZATION_OGRN,
+                    installation_name=settings.RDM_UPLOADER_CLIENT_INSTALLATION_NAME,
+                    installation_id=settings.RDM_UPLOADER_CLIENT_INSTALLATION_ID,
+                    username=settings.RDM_UPLOADER_CLIENT_USERNAME,
+                    password=settings.RDM_UPLOADER_CLIENT_PASSWORD,
+                )
+            )
+        else:
+            uploader_client.set_config(
+                RegionalDataMartUploaderConfig(
+                    url=settings.RDM_UPLOADER_CLIENT_URL,
+                    datamart_name=settings.RDM_UPLOADER_CLIENT_DATAMART_NAME,
+                    timeout=settings.RDM_UPLOADER_CLIENT_REQUEST_TIMEOUT,
+                    request_retries=settings.RDM_UPLOADER_CLIENT_REQUEST_RETRIES,
+                )
+            )
+
+    def upload_data(self, *args, **kwargs):
+        """
+        Запускает отправку данных в витрину.
+        """
+        try:
+            exporter = ExportEntityQueueSender(self.data_cache, self.queue, self.entities)
+            exporter.run()
+
+            sender = WorkerSender(self.queue)
+            sender.run()
+            
+        except Exception as err:
+            logger.exception(err)
+            raise err
+        finally:
+            self._remove_file_handler()

edu_rdm_integration/export_data/export_manger.py

@@ -1,10 +1,8 @@
-from datetime import (
-    datetime,
-)
 from pathlib import (
     Path,
 )
 from typing import (
+    Iterable,
     List,
 )
 
@@ -13,6 +11,7 @@ from django.conf import (
 )
 from django.db.models import (
     F,
+    Q,
     Sum,
 )
 from django.db.transaction import (
@@ -83,10 +82,14 @@ class ExportQueueSender:
         return file_size or 0
 
     @staticmethod
-    def get_sub_stages_attachments_to_export():
+    def _make_stage_filter(self) -> Q:
+        """Формирование фильтра для выборки подэтапов."""
+        return Q(status_id=ExportingDataSubStageStatus.READY_FOR_EXPORT.key)
+
+    def get_sub_stages_attachments_to_export(self):
         """Выборка готовых к экспорту подэтапов."""
         return ExportingDataSubStage.objects.filter(
-            status_id=ExportingDataSubStageStatus.READY_FOR_EXPORT.key
+            self._make_stage_filter()
         ).annotate(
             attachment_id=F('exportingdatasubstageattachment__id'),
             attachment_name=F('exportingdatasubstageattachment__attachment'),
@@ -117,10 +120,14 @@ class ExportQueueSender:
             timeout=settings.RDM_REDIS_CACHE_TIMEOUT_SECONDS
         )
 
+        logger.info(
+            f'ExportedDataSubStage {sub_stage_id} {entity_name} added to the queue'
+        )
+
         return True
 
     def run(self):
-        """Запуск работы менджера."""
+        """Запуск работы очереди."""
         if not self.queue_total_file_size:
             self.queue_total_file_size = self.get_exported_file_size()
 
@@ -133,9 +140,7 @@ class ExportQueueSender:
                 if prev_sub_stage != stage_attachment['id']:
                     if stage_files:
                         to_cache = self.set_sub_stage_to_cache(prev_sub_stage, entity, stage_files)
-                        logger.info(
-                            f'ExportedDataSubStage {prev_sub_stage} {entity} added to the queue'
-                        )
+
                         stage_files = []
 
                         if not to_cache:
@@ -162,6 +167,25 @@ class ExportQueueSender:
                 self.queue_total_file_size,
                 timeout=settings.RDM_REDIS_CACHE_TIMEOUT_SECONDS
             )
+            logger.warning(
+                f'Total exported file size:  {self.queue_total_file_size} - queue is full!!!'
+            )
+
+
+class ExportEntityQueueSender(ExportQueueSender):
+    """Класс отправки данных по конкретной сущности в очередь РВД."""
+
+    def __init__(self, data_cache: AbstractCache, queue: Queue, entity: Iterable[str]):
+        super().__init__(data_cache, queue)
+
+        self.entity = entity
+
+    def _make_stage_filter(self) -> Q:
+        """Формирование фильтра для выборки подэтапов."""
+        return Q(
+            status_id=ExportingDataSubStageStatus.READY_FOR_EXPORT.key,
+            exportingdatasubstageentity__entity_id__in=self.entity
+        )
 
 
 class WorkerSender:
@@ -240,6 +264,9 @@ class WorkerSender:
                     sub_stage.save()
 
             self.queue.delete_from_queue(sub_stage_id=sub_stage_id, entity_name=entity_key)
+            logger.info(
+                f'ExportedDataSubStage {sub_stage_id} {entity_key} sended from the queue'
+            )
 
     def run(self):
         """Запуск воркера отправки."""

edu_rdm_integration/helpers.py

@@ -10,8 +10,8 @@ from typing import (
     TYPE_CHECKING,
     Any,
     Dict,
-    List,
     Iterable,
+    List,
     Optional,
     Tuple,
     Union,
@@ -33,16 +33,16 @@ from django.db.models.functions import (
     Cast,
     Least,
 )
+from uploader_client.adapters import (
+    adapter,
+)
 
 from educommon import (
     logger,
 )
-from uploader_client.adapters import (
-    adapter,
-)
+
 from edu_rdm_integration.collect_and_export_data.models import (
-    EduRdmCollectDataCommandProgress,
-    EduRdmExportDataCommandProgress,
+    AbstractExportDataCommandProgress,
 )
 from edu_rdm_integration.enums import (
     FileUploadStatusEnum,
@@ -54,32 +54,35 @@ from edu_rdm_integration.export_data.consts import (
     TOTAL_ATTACHMENTS_SIZE_KEY,
 )
 from edu_rdm_integration.models import (
-    DataMartRequestStatus,
-    ExportingDataStage,
     CollectingDataStageStatus,
     CollectingExportedDataStage,
+    DataMartRequestStatus,
+    ExportingDataStage,
+    ExportingDataStageStatus,
     ExportingDataSubStageUploaderClientLog,
+    UploadDataCommand,
     UploadStatusRequestLog,
-    ExportingDataStageStatus,
 )
 from edu_rdm_integration.redis_cache import (
     AbstractCache,
 )
 
+
 if TYPE_CHECKING:
     from datetime import (
         datetime,
     )
 
+    from uploader_client.models import (
+        Entry,
+    )
+
     from edu_rdm_integration.collect_data.non_calculated.base.managers import (
         BaseCollectingExportedDataRunnerManager,
     )
     from edu_rdm_integration.export_data.base.managers import (
         BaseExportDataRunnerManager,
     )
-    from uploader_client.models import (
-        Entry,
-    )
 
 
 class UploadStatusHelper:
@@ -311,7 +314,7 @@ class Graph:
 
 
 def save_command_log_link(
-    command: Union[EduRdmCollectDataCommandProgress, EduRdmExportDataCommandProgress],
+    command: Union[AbstractExportDataCommandProgress, UploadDataCommand],
     log_dir: str
 ) -> None:
     """Сохраняет ссылку на лог команды."""

edu_rdm_integration/mapping.py

@@ -14,3 +14,6 @@ MODEL_FIELDS_LOG_FILTER: Dict[EntityLogOperation, Dict[str, Tuple]] = {
     EntityLogOperation.UPDATE: {},
     EntityLogOperation.DELETE: {}
 }
+
+# Маппинг операций логов моделей и сущностей по умолчанию
+DEFAULT_ENTITY_LOG_OPERATION_MAP = {op: op for op in EntityLogOperation.values}

edu_rdm_integration/migrations/0014_uploaddatacommand.py

@@ -0,0 +1,43 @@
+# Generated by Django 3.1.14 on 2024-09-05 20:35
+
+import uuid
+
+import django.db.models.deletion
+import django.utils.timezone
+from django.db import (
+    migrations,
+    models,
+)
+
+import educommon.django.db.mixins
+
+import edu_rdm_integration.utils
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('async_task', '0002_task_type_and_status_data'),
+        ('edu_rdm_integration', '0013_set_attachment_size'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='UploadDataCommand',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('logs_link', models.FileField(max_length=255, upload_to=edu_rdm_integration.utils.get_data_command_progress_attachment_path, verbose_name='Ссылка на файл логов')),
+                ('type', models.PositiveSmallIntegerField(choices=[(1, 'Автоматический'), (2, 'Ручной')], verbose_name='Тип команды')),
+                ('created', models.DateTimeField(default=django.utils.timezone.now, verbose_name='Дата создания')),
+                ('generation_id', models.UUIDField(default=uuid.uuid4, verbose_name='Идентификатор генерации')),
+                ('entity', models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to='edu_rdm_integration.regionaldatamartentityenum', verbose_name='Сущность РВД')),
+                ('task', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to='async_task.runningtask', verbose_name='Асинхронная задача')),
+            ],
+            options={
+                'verbose_name': 'Команда отправки данных в витрину',
+                'verbose_name_plural': 'Команды отправки данных в витрину',
+                'db_table': 'rdm_upload_data_command',
+            },
+            bases=(educommon.django.db.mixins.ReprStrPreModelMixin, models.Model),
+        ),
+    ]

edu_rdm_integration/models.py

@@ -26,6 +26,7 @@ from django.db.models import (
     Manager,
     OneToOneField,
     PositiveIntegerField,
+    PositiveSmallIntegerField,
     SmallIntegerField,
     UUIDField,
 )
@@ -65,12 +66,14 @@ from m3_db_utils.models import (
 )
 
 from edu_rdm_integration.enums import (
+    CommandType,
     FileUploadStatusEnum,
 )
 from edu_rdm_integration.uploader_log.managers import (
     UploaderClientLogManager,
 )
 from edu_rdm_integration.utils import (
+    get_data_command_progress_attachment_path,
     get_exporting_data_stage_attachment_path,
 )
 
@@ -150,6 +153,7 @@ class CollectingExportedDataStage(ReprStrPreModelMixin, BaseObjectModel):
 
     @property
     def attrs_for_repr_str(self):
+        """Список атрибутов для отображения экземпляра модели."""
         return ['manager_id', 'logs_period_started_at', 'logs_period_ended_at', 'started_at', 'ended_at', 'status_id']
 
     def save(self, *args, **kwargs):
@@ -245,6 +249,7 @@ class CollectingExportedDataSubStage(ReprStrPreModelMixin, BaseObjectModel):
 
     @property
     def attrs_for_repr_str(self):
+        """Список атрибутов для отображения экземпляра модели."""
         return ['stage_id', 'function_id', 'started_at', 'ended_at', 'previous_id', 'status_id']
 
     def save(self, *args, **kwargs):
@@ -335,6 +340,7 @@ class ExportingDataStage(ReprStrPreModelMixin, BaseObjectModel):
 
     @property
     def attrs_for_repr_str(self):
+        """Список атрибутов для отображения экземпляра модели."""
         return ['manager_id', 'started_at', 'ended_at', 'status_id']
 
     def save(self, *args, **kwargs):
@@ -421,9 +427,11 @@ class ExportingDataSubStage(ReprStrPreModelMixin, BaseObjectModel):
 
     @property
     def attrs_for_repr_str(self):
+        """Список атрибутов для отображения экземпляра модели."""
         return ['function_id', 'collecting_data_sub_stage_id', 'stage_id', 'started_at', 'ended_at', 'status_id']
 
     def save(self, *args, **kwargs):
+        """Сохранение экземпляра модели."""
         if (
             self.status_id in {
                 ExportingDataSubStageStatus.FAILED.key,
@@ -487,6 +495,7 @@ class ExportingDataSubStageAttachment(ReprStrPreModelMixin, BaseObjectModel):
 
     @property
     def attrs_for_repr_str(self):
+        """Список атрибутов для отображения экземпляра модели."""
         return ['exporting_data_sub_stage_id', 'attachment', 'operation', 'created', 'modified']
 
 
@@ -675,6 +684,7 @@ class BaseEntityModel(ReprStrPreModelMixin, BaseObjectModel):
 
     @property
     def attrs_for_repr_str(self):
+        """Список атрибутов для отображения экземпляра модели."""
         return ['collecting_sub_stage', 'exporting_sub_stage', 'operation', 'created', 'modified']
 
     class Meta:
@@ -931,6 +941,7 @@ class TransferredEntity(BaseObjectModel):
 
     @json_encode
     def no_export(self):
+        """Формирует отображение признака отключения экспорта."""
         return 'Нет' if self.export_enabled else 'Да'
 
 
@@ -953,3 +964,42 @@ class ExportingDataSubStageEntity(BaseObjectModel):
         db_table = 'rdm_exporting_data_sub_stage_entity'
         verbose_name = 'Связь сущности и подэтапа выгрузки'
         verbose_name_plural = 'Связи сущности и подэтапа выгрузки'
+
+
+class UploadDataCommand(ReprStrPreModelMixin, BaseObjectModel):
+    """Модель, хранящая данные для формирования и отслеживания асинхронной задачи по отправке данных в витрину."""
+
+    task = ForeignKey(
+        to='async_task.RunningTask',
+        verbose_name='Асинхронная задача',
+        blank=True,
+        null=True,
+        on_delete=SET_NULL,
+    )
+    logs_link = FileField(
+        upload_to=get_data_command_progress_attachment_path,
+        max_length=255,
+        verbose_name='Ссылка на файл логов',
+    )
+    type = PositiveSmallIntegerField(  # noqa: A003
+        verbose_name='Тип команды',
+        choices=CommandType.get_choices(),
+    )
+    entity = ForeignKey(
+        to=RegionalDataMartEntityEnum,
+        verbose_name='Сущность РВД',
+        on_delete=PROTECT,
+    )
+    created = DateTimeField(
+        verbose_name='Дата создания',
+        default=timezone.now,
+    )
+    generation_id = UUIDField(
+        'Идентификатор генерации',
+        default=uuid.uuid4,
+    )
+
+    class Meta:
+        db_table = 'rdm_upload_data_command'
+        verbose_name = 'Команда отправки данных в витрину'
+        verbose_name_plural = 'Команды отправки данных в витрину'

edu_rdm_integration/tasks.py

@@ -9,9 +9,6 @@ from typing import (
 )
 
 import celery
-from celery.exceptions import (
-    Ignore,
-)
 from celery.schedules import (
     crontab,
 )
@@ -25,9 +22,6 @@ from django.utils import (
     timezone,
 )
 
-from educommon.async_task.locker import (
-    TaskLocker,
-)
 from educommon.async_task.models import (
     AsyncTaskType,
     RunningTask,
@@ -64,10 +58,7 @@ from edu_rdm_integration.enums import (
 )
 from edu_rdm_integration.export_data.export import (
     ExportLatestEntitiesData,
-)
-from edu_rdm_integration.export_data.export_manger import (
-    ExportQueueSender,
-    WorkerSender,
+    UploadEntitiesData,
 )
 from edu_rdm_integration.export_data.helpers import (
     set_failed_status_suspended_exporting_data_stages,
@@ -85,6 +76,7 @@ from edu_rdm_integration.models import (
     ExportingDataSubStageUploaderClientLog,
     RegionalDataMartEntityEnum,
     TransferredEntity,
+    UploadDataCommand,
 )
 from edu_rdm_integration.storages import (
     RegionalDataMartEntityStorage,
@@ -354,6 +346,7 @@ class TransferLatestEntitiesDataPeriodicTask(UniquePeriodicAsyncTask):
 
 class UploadDataAsyncTask(PeriodicAsyncTask):
     """Формирование очереди файлов и их отправка."""
+
     queue = TASK_QUEUE_NAME
     routing_key = TASK_QUEUE_NAME
     description = 'Отправка данных в витрину "Региональная витрина данных"'
@@ -369,11 +362,31 @@ class UploadDataAsyncTask(PeriodicAsyncTask):
         """Выполнение."""
         super().process(*args, **kwargs)
 
+        entity_ids_for_export = list(
+            TransferredEntity.objects.filter(export_enabled=True).values_list('entity_id', flat=True)
+        )
         queue = RdmRedisSubStageAttachmentQueue()
-        exporter = ExportQueueSender(cache, queue)
-        exporter.run()
-        sender = WorkerSender(queue)
-        sender.run()
+        task_id = RunningTask.objects.filter(
+            pk=self.request.id,
+        ).values_list('pk', flat=True).first()
+
+        if task_id:
+            for entity_id in entity_ids_for_export:
+                upload_data_command = UploadDataCommand.objects.create(
+                    entity_id=entity_id,
+                    task_id=task_id,
+                    type=CommandType.AUTO,
+                )
+                if upload_data_command:
+                    upload_data = UploadEntitiesData(
+                        entities=[upload_data_command.entity_id],
+                        data_cache=cache,
+                        queue=queue,
+                        task_id=task_id,
+                        command_id=upload_data_command.id,
+                    )
+                    upload_data.upload_data()
+                    save_command_log_link(upload_data_command, settings.RDM_UPLOAD_LOG_DIR)
 
 
 celery_app = celery.app.app_or_default()
@@ -381,4 +394,3 @@ celery_app.register_task(RDMCheckUploadStatus)
 celery_app.register_task(CheckSuspendedExportedStagePeriodicTask)
 celery_app.register_task(TransferLatestEntitiesDataPeriodicTask)
 celery_app.register_task(UploadDataAsyncTask)
-

edu_rdm_integration/typing.py

@@ -0,0 +1,28 @@
+from typing import (
+    Sequence,
+    TypeVar,
+    Union,
+)
+
+from django.db.models import (
+    Expression,
+    Model,
+)
+
+
+# Тип, обозначающий любую модель. При указании в качестве type annotation
+# можно указать, что аргументом может быть любая модель или тип модели (через
+# Type[MODEL_TYPE_VAR]), а сама функция возвращает инстанс этой
+# конкретной модели
+MODEL_TYPE_VAR = TypeVar('MODEL_TYPE_VAR', bound=Model)
+
+# Аннотация типов для id записи в БД
+RECORD_IDS = Union[
+    tuple[int, ...],
+    tuple[str, ...],
+    list[int],
+    list[str],
+    Sequence[int],
+    Sequence[str],
+    Expression,
+]

edu_rdm_integration/utils.py

@@ -1,5 +1,6 @@
 import os
 from datetime import (
+    date,
     datetime,
     time,
     timedelta,
@@ -11,12 +12,30 @@ from typing import (
     List,
     Optional,
     Set,
+    Type,
+    TypeVar,
     Union,
 )
 
 from django.conf import (
     settings,
 )
+from django.db.models import (
+    DateField,
+    DateTimeField,
+    Expression,
+    F,
+)
+from django.db.models.expressions import (
+    Func,
+)
+from django.db.models.functions import (
+    Cast,
+    Now,
+)
+from educommon.integration_entities.enums import (
+    EntityLogOperation,
+)
 
 from edu_rdm_integration.apps import (
     EduRDMIntegrationConfig,
@@ -285,3 +304,59 @@ def get_data_command_progress_attachment_path(
         str(instance.type),
         filename
     )
+
+
+class MakeInterval(Func):
+    """Функция обработки даты/времени."""
+
+    template = 'make_interval(%(expressions)s)'
+
+    def __init__(
+        self, *,
+        years: Union[int, F] = 0,
+        months: Union[int, F] = 0,
+        weeks: Union[int, F] = 0,
+        days: Union[int, F] = 0,
+        hours: Union[int, F] = 0,
+        minutes: Union[int, F] = 0,
+        seconds: Union[float, F] = 0.0,
+        output_field=None,
+    ) -> None:
+        self.years = years
+        self.months = months
+        self.weeks = weeks
+        self.days = days
+        self.hours = hours
+        self.minutes = minutes
+        self.seconds = seconds
+
+        super().__init__(
+            self.years,
+            self.months,
+            self.weeks,
+            self.days,
+            self.hours,
+            self.minutes,
+            self.seconds,
+            output_field=output_field or DateTimeField(),
+        )
+
+
+def make_passed_datetime_from_today(
+    *,
+    years: int = 0,
+    months: int = 0,
+    days: int = 0,
+) -> Union[datetime, Expression, F]:
+    """Возвращает выражение на лет/месяцев/дней меньше относительно сегодня."""
+    now_expr = Now()
+
+    return now_expr - MakeInterval(years=years, months=months, days=days)
+
+
+TODAY_EXPR: Union[date, Expression, F] = Cast(make_passed_datetime_from_today(), output_field=DateField())
+
+THREE_YEARS_AGO_DATE_EXPR: Union[date, Expression, F] = Cast(
+    make_passed_datetime_from_today(years=3),
+    output_field=DateField()
+)

{edu_rdm_integration-3.3.6.dist-info → edu_rdm_integration-3.4.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: edu-rdm-integration
-Version: 3.3.6
+Version: 3.4.0
 Summary: Интеграция с Региональной витриной данных
 Home-page: 
 Download-URL: 
@@ -28,7 +28,7 @@ Requires-Dist: wheel <0.42,>=0.37.1
 Requires-Dist: transliterate <2
 Requires-Dist: Django <3.3,>=2.2
 Requires-Dist: celery <5.3,>=4.4.7
-Requires-Dist: educommon <4,>=3.10.0
+Requires-Dist: educommon <4,>=3.11.0
 Requires-Dist: function-tools <1,>=0.9.0
 Requires-Dist: m3-db-utils <1,>=0.3.10
 Requires-Dist: m3-django-compat <2,>=1.10.2
@@ -301,6 +301,29 @@ Requires-Dist: uploader-client <1,>=0.2.1
 
 ### Удалено
 
+
+## 3.4.0 - 2024-09-25
+В базовый класс тестирования функций сбора добавлена поддержка формата PhoneNumber для формирования
+экземпляра модели AuditLog.
+Повышена версия зависимости educommon >= 3.11.0.
+Вынесены утилиты и базовый класс для логирования и сбора по модели в рамках сущности
+
+## Добавлено
+- [EDUSCHL-22378](https://jira.bars.group/browse/EDUSCHL-22378)
+  Реализация поля моделей Django содержащего номер телефона
+
+### Изменено
+- [EDUKNDG-14516](https://jira.bars.group/browse/EDUKNDG-14516)
+   PATCH Вынесены утилиты и базовый класс для логирования и сбора по модели в рамках сущности
+
+
+## 3.3.7 - 2024-09-06
+Добавлены команда (UploadEntitiesData) и модель (UploadDataCommand) для логирования и запуска выгрузки данных в витрину
+- [EDUSCHL-22042](https://jira.bars.group/browse/EDUSCHL-22042)
+  PATCH Добавлены команда (UploadEntitiesData) и модель (UploadDataCommand) для логирования и запуска выгрузки данных 
+в витрину
+
+
 ## 3.3.6 - 2024-09-02
 Добавлена фильтрация по дате выгрузки при обновлении данных на экспорт в BaseExportLatestEntitiesData
 
@@ -309,7 +332,6 @@ Requires-Dist: uploader-client <1,>=0.2.1
    PATCH Добавлена фильтрация по дате выгрузки при обновлении данных на экспорт в BaseExportLatestEntitiesData
 
 
-
 ## 3.3.5 - 2024-08-26
 Классы UniquePeriodicAsyncTask, PeriodicTaskLocker перенесены в educommon.
 

{edu_rdm_integration-3.3.6.dist-info → edu_rdm_integration-3.4.0.dist-info}/RECORD

@@ -6,14 +6,15 @@ edu_rdm_integration/base.py,sha256=_G0qPTAXe6bXfgDHNiZMSsYt3sMuUhLKnHuQCWSFttU,1
 edu_rdm_integration/consts.py,sha256=Qt52SOCQ-3wOet-_6inJih_W9nToORKXbkxb3jVSjEo,1079
 edu_rdm_integration/entities.py,sha256=qNVWUhjwvX298Ak86_AKmqBZioP0czGwBcAz_4dtUUE,14552
 edu_rdm_integration/enums.py,sha256=T3Mu5D-CbKO3BSg16MPPnIPlcc_YGLYR-ThS8dzl9gg,4246
-edu_rdm_integration/helpers.py,sha256=m98k8nichw5AIoPuXNPsI-ZFN9OYX5Wt2MU6ml_pg88,14615
-edu_rdm_integration/mapping.py,sha256=bwa2fJCbV4YjQcAgRrgT3hgM6dJhr_uBtQgx3L3F2Ck,473
-edu_rdm_integration/models.py,sha256=Fwt2O2j_aPI9owRZprreGQ2t7op-YmIYpZoq5HHjL3M,30692
+edu_rdm_integration/helpers.py,sha256=_zQb3vuMrWClpUErshYTI1aVgdKjXf819pHfBjaIty4,14592
+edu_rdm_integration/mapping.py,sha256=PGy6oH3Jzg4uuSIjlI0A-gWWpJPozqZHN4wGzp-ldFY,660
+edu_rdm_integration/models.py,sha256=YP_vot6PbQ3jWv3NNW1wa0gScjOsT2lgdrj7z5V6NHw,33053
 edu_rdm_integration/redis_cache.py,sha256=GZhtM1d0cVr5TEqxh15K7dS371Msit6wRemIiYb2rzk,1548
 edu_rdm_integration/signals.py,sha256=3eRlpkDcFCF6TN80-QM8yBYLcyozzcmoPjz6r4_ApWg,73
 edu_rdm_integration/storages.py,sha256=o5WqUG7SnkeuMt-z8spUi-IraivST-7KHzfY-M3v7FA,6807
-edu_rdm_integration/tasks.py,sha256=WolNe2t3_NCOisZeg24HyPI3Y0ciOJbwaFprhWgt41Q,15282
-edu_rdm_integration/utils.py,sha256=NIOxlH4JiTOOxYgqcwtfqT5UAtNG24GLq_tsnBUtp8E,10370
+edu_rdm_integration/tasks.py,sha256=QAOytSCEoWtFWJqmTkj3N6NRl_Hds9EXyRfngFHrbPk,16046
+edu_rdm_integration/typing.py,sha256=6GOZfRDqOjN-o5NR86-f2o4uXQm_8AUCRRdQTITtrcs,785
+edu_rdm_integration/utils.py,sha256=QMiohtOaId3Zab91IZPazGzhce5pa5e-ilRgXpISkHk,12249
 edu_rdm_integration/adapters/__init__.py,sha256=cU0swn4Ny5ZQz5buWRcWsT1mpWuUFJaUlHf2l7TtEBo,83
 edu_rdm_integration/adapters/apps.py,sha256=TyJTkSPs2qAHJ11fqbwLGk3Ea7ujtqWwbxqmvYNQxG8,363
 edu_rdm_integration/adapters/caches.py,sha256=OxSqeXySUN42LxEeHBLtC1ZBt-7aicbRbmP1EJYTvV4,1505
@@ -41,7 +42,7 @@ edu_rdm_integration/collect_data/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRk
 edu_rdm_integration/collect_data/collect.py,sha256=HIKO8Kvw9NY9Nt5M5w9XfcZ9E5IVue_4mT7ESYJ6n4I,11909
 edu_rdm_integration/collect_data/generators.py,sha256=f34AAwdEcQNIokX0ypqYgjRD1XolwBVLER_HYv9ibNw,9075
 edu_rdm_integration/collect_data/helpers.py,sha256=gAFZAm9YhMtQhvlwSF3LMthPPa8LsqG_zbVe7vnW_Ag,2995
-edu_rdm_integration/collect_data/tests.py,sha256=-UNX3-GL0H9i89GXRfICT0ABltn_9aN_I_cmn1gQcDA,5367
+edu_rdm_integration/collect_data/tests.py,sha256=hlf2eXUQonYeLY2enm-ND712NrsGcYa03FxjCUOj7EE,5535
 edu_rdm_integration/collect_data/base/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 edu_rdm_integration/collect_data/base/caches.py,sha256=XPMVZSgR8vRhCHmHsnUKy86rEIPUQXHz0wJabovPyfU,2170
 edu_rdm_integration/collect_data/base/functions.py,sha256=E1vYmC8F8NM5cBB5SGQEpluHrMynpjf3Ek3s0HsbxwY,2998
@@ -82,13 +83,13 @@ edu_rdm_integration/collect_data/non_calculated/base/strings.py,sha256=-k9dex8A7
 edu_rdm_integration/collect_data/non_calculated/base/tests.py,sha256=MoRY-a75Ow-7EjeQYxkXWunwqTGuBMaUyEkEV2oy05I,59
 edu_rdm_integration/collect_data/non_calculated/base/validators.py,sha256=0YvnfrfK1iFcZVSB-M-Xv82tIjYxEU_BwLofAEuGVW4,973
 edu_rdm_integration/enum_register/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-edu_rdm_integration/enum_register/mixins.py,sha256=nmHue9-9bqKZn9_MFiL0E-5wBUBIZPZwk3DWVir5pKk,4021
+edu_rdm_integration/enum_register/mixins.py,sha256=YYCvSQEf6RMaMnGS2vA2v-P1ojkXA5MgAuBOcFgLIRE,4642
 edu_rdm_integration/enum_register/register.py,sha256=5OWOjK-M0Erd_5CENpBaXhVtfL0pEaDl3Bev5QKNDJc,2218
 edu_rdm_integration/export_data/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 edu_rdm_integration/export_data/consts.py,sha256=rmJ_18wHCE54j0VABxfo7Zu01EPFYSjYrj5L79hVW2Y,333
 edu_rdm_integration/export_data/dataclasses.py,sha256=IhftRopP4lS-m3ygdBU5Bz0HF71VSBP4JQ6-8VIVgtY,260
-edu_rdm_integration/export_data/export.py,sha256=JZAgT8K49buRb57Pvt7iRkKW_fm7P4luZ4_umbsoY_E,16609
-edu_rdm_integration/export_data/export_manger.py,sha256=_8gkFH380zZVJ9LVxqeZOXphS-RjeIoSbq0eUbXnIMA,9657
+edu_rdm_integration/export_data/export.py,sha256=xR2QRRL_HLD9MUIoe98Krc4KN0itWCpEl3uDM2tcqeE,18513
+edu_rdm_integration/export_data/export_manger.py,sha256=lbXK3aV0Te0ZfJHylvCBZ0bJ7vWwD-pMQBX4Du8gPro,10681
 edu_rdm_integration/export_data/generators.py,sha256=yLDOcHB1PoilJwXtKGxZQhDjpeKBzEWoosahbJJ4Ba4,4020
 edu_rdm_integration/export_data/helpers.py,sha256=hU346RmQ17Ra2etFvxXI7JQlLyp_0KxH1jm-eeCqejc,2933
 edu_rdm_integration/export_data/queue.py,sha256=NiWm7e59zOcGT8s87BxiyU4-nA02fH94oVKnhc8htQ4,6159
@@ -162,6 +163,7 @@ edu_rdm_integration/migrations/0010_transferredentity_export_enabled.py,sha256=L
 edu_rdm_integration/migrations/0011_exportingdatasubstageentity.py,sha256=2BfIif_hkFv1h6VEfe0Ys4J_uk6LR9YtO711ocDYPso,1263
 edu_rdm_integration/migrations/0012_exportingdatasubstageattachment_attachment_size.py,sha256=y_JQO69k9pEfrJyimaRiAOBmhaJmssIyepCGd-Sy9hs,511
 edu_rdm_integration/migrations/0013_set_attachment_size.py,sha256=Pj_n-ytsC0lhyU67qvH8UHHQ-c-TH5MZFfy-UF6y6M4,1809
+edu_rdm_integration/migrations/0014_uploaddatacommand.py,sha256=Hh0vKKiGgKOvY1kBAcmway4dSYUXwVArHAc9YrsjCIU,2079
 edu_rdm_integration/migrations/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 edu_rdm_integration/registry/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 edu_rdm_integration/registry/actions.py,sha256=YUtCkpcc3DedM_x8rwJ6Aos-8HKkDCrOUdKnGhztfUU,5223
@@ -174,9 +176,9 @@ edu_rdm_integration/uploader_log/enums.py,sha256=rgSO3BL2rh2xpfm0Pt4waQW8fB1VMJL
 edu_rdm_integration/uploader_log/managers.py,sha256=y5wTSMzF9hpOpIU_A7nIafL_LBU3QEie6LAYWoB-pBQ,3203
 edu_rdm_integration/uploader_log/ui.py,sha256=YM9Buqp2wxE95Wf5gvAATBzuYzDOossK1sEmvFk07cI,2110
 edu_rdm_integration/uploader_log/templates/ui-js/object-grid-buttons.js,sha256=2xyGe0wdVokM0RhpzRzcRvJPBkBmPe3SlZry4oP4Nzs,6201
-edu_rdm_integration-3.3.6.dist-info/LICENSE,sha256=uw43Gjjj-1vXWCItfSrNDpbejnOwZMrNerUh8oWbq8Q,3458
-edu_rdm_integration-3.3.6.dist-info/METADATA,sha256=GF0r7I7EfNUKBqQ4USLovvZRvmn4gAJzDxG8KPeN81I,72243
-edu_rdm_integration-3.3.6.dist-info/WHEEL,sha256=yQN5g4mg4AybRjkgi-9yy4iQEFibGQmlz78Pik5Or-A,92
-edu_rdm_integration-3.3.6.dist-info/namespace_packages.txt,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
-edu_rdm_integration-3.3.6.dist-info/top_level.txt,sha256=nRJV0O14UtNE-jGIYG03sohgFnZClvf57H5m6VBXe9Y,20
-edu_rdm_integration-3.3.6.dist-info/RECORD,,
+edu_rdm_integration-3.4.0.dist-info/LICENSE,sha256=uw43Gjjj-1vXWCItfSrNDpbejnOwZMrNerUh8oWbq8Q,3458
+edu_rdm_integration-3.4.0.dist-info/METADATA,sha256=yyTvi9MApMnWTUi9cv5G3MRQ8CKcEb48KFVSvcGQXzk,73632
+edu_rdm_integration-3.4.0.dist-info/WHEEL,sha256=yQN5g4mg4AybRjkgi-9yy4iQEFibGQmlz78Pik5Or-A,92
+edu_rdm_integration-3.4.0.dist-info/namespace_packages.txt,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
+edu_rdm_integration-3.4.0.dist-info/top_level.txt,sha256=nRJV0O14UtNE-jGIYG03sohgFnZClvf57H5m6VBXe9Y,20
+edu_rdm_integration-3.4.0.dist-info/RECORD,,