edu-rdm-integration 3.5.2__py3-none-any.whl → 3.5.4__py3-none-any.whl

edu_rdm_integration/app_settings.py

@@ -52,8 +52,11 @@ RDM_UPLOAD_QUEUE_MAX_SIZE = 500_000_000
 # Таймаут для сохранения параметров в общем кеш
 RDM_REDIS_CACHE_TIMEOUT_SECONDS = 60 * 60 * 2
 
+
 # Настройка запуска периодической задачи отправки csv-файлов в витрину:
 RDM_UPLOAD_DATA_TASK_MINUTE = '0'
 RDM_UPLOAD_DATA_TASK_HOUR = '*/2'
 RDM_UPLOAD_DATA_TASK_DAY_OF_WEEK = '*'
-RDM_UPLOAD_DATA_TASK_LOCK_EXPIRE_SECONDS = 60 * 60 * 2
+RDM_UPLOAD_DATA_TASK_LOCK_EXPIRE_SECONDS = 60 * 60 * 2
+# Количество подэтапов для обработки в периодической задаче отправки данных
+RDM_UPLOAD_DATA_TASK_EXPORT_STAGES = 500

edu_rdm_integration/collect_data/base/mixins.py

@@ -173,7 +173,8 @@ class BaseIgnoreLogMixin:
     def _exclude_logs(self, model_label: str, object_model_ids: Iterable[int]):
         """Исключаем логи из обработки."""
         for object_id in object_model_ids:
-            self.logs[model_label].pop(object_id)
+            if object_id in self.logs[model_label]:
+                del self.logs[model_label][object_id]
 
     def _ignore_logs(self):
         """

edu_rdm_integration/export_data/consts.py

@@ -4,3 +4,6 @@ DELIMITER = ';'
 REDIS_QUEUE_KEY_DELIMITER = ':'
 # Ключ параметра Общий объем отправленных в РВД файлов
 TOTAL_ATTACHMENTS_SIZE_KEY = 'rdm:total_attachments_size'
+
+# Ограничение количества файлов на обработку
+ATTACHMENTS_LIMIT = 500

edu_rdm_integration/export_data/export.py

@@ -1,3 +1,4 @@
+import logging
 import os
 from collections import (
     defaultdict,
@@ -67,7 +68,7 @@ from edu_rdm_integration.enums import (
     FileUploadStatusEnum,
 )
 from edu_rdm_integration.export_data.export_manger import (
-    ExportEntityQueueSender,
+    ExportQueueSender,
     WorkerSender,
 )
 from edu_rdm_integration.export_data.queue import (
@@ -370,39 +371,41 @@ class ExportLatestEntitiesData(BaseExportLatestEntitiesData):
         return RunningTask
 
 
-class UploadEntitiesData(BaseOperationData):
-    """Класс отправки файлов с сущностями в витрину."""
+class UploadData(BaseOperationData):
+    """Класс отправки файлов в витрину."""
 
     def __init__(
         self,
-        entities: Iterable[str],
         data_cache: AbstractCache,
         queue: Queue,
         **kwargs,
     ):
         super().__init__(**kwargs)
 
-        # Если сущности не указаны, берется значение по умолчанию - все сущности:
-        self.entities = entities if entities else RegionalDataMartEntityEnum.get_enum_data().keys()
-
         self.data_cache = data_cache
         self.queue = queue
 
-        self.task_id = kwargs.get('task_id')
-
         self._configure_agent_client()
+        self.result = {
+            'total_file_size': 0,  # Общий размер отправленных файлов
+            'queue_is_full': False,  # Признак переполнения очереди
+            'uploaded_entities': ''  # Список сущностей, попавших в выгрузку
+        }
 
     @property
     def _log_file_path(self) -> Union[str, bytes]:
         """
         Путь до лог файла.
         """
-        if self.command_id:
-            log_file_path = os.path.join(settings.MEDIA_ROOT, settings.RDM_UPLOAD_LOG_DIR, f'{self.command_id}.log')
-        else:
-            log_file_path = os.path.join(settings.MEDIA_ROOT, settings.RDM_UPLOAD_LOG_DIR, 'upload_entity.log')
+        return os.path.join(settings.MEDIA_ROOT, settings.RDM_UPLOAD_LOG_DIR, 'upload_entity.log')
 
-        return log_file_path
+    def _add_file_handler(self) -> None:
+        """Добавляет обработчик логов."""
+
+        self._file_handler = logging.FileHandler(self._log_file_path)
+
+        logging.getLogger('info_logger').addHandler(self._file_handler)
+        logging.getLogger('exception_logger').addHandler(self._file_handler)
 
     # TODO https://jira.bars.group/browse/EDUSCHL-22492. Вынужденная мера, т.к. при запуске команды не производится
     #  проверка готовности конфигов приложений. Нужно переработать механизм конфигурирования клиента загрузчика.
@@ -459,14 +462,24 @@ class UploadEntitiesData(BaseOperationData):
         Запускает отправку данных в витрину.
         """
         try:
-            exporter = ExportEntityQueueSender(self.data_cache, self.queue, self.entities)
+            exporter = ExportQueueSender(self.data_cache, self.queue, settings.RDM_UPLOAD_DATA_TASK_EXPORT_STAGES)
             exporter.run()
 
-            sender = WorkerSender(self.queue)
-            sender.run()
-            
+            self.result['queue_is_full'] = exporter.queue_is_full
+            self.result['total_file_size'] = exporter.queue_total_file_size
+
+            # Если очередь не переполнена - то отправляем данные в витрину
+            if not exporter.queue_is_full:
+                sender = WorkerSender(self.queue)
+                sender.run()
+
+                if sender.entities:
+                    self.result['uploaded_entities'] = ','.join(sender.entities)
+
         except Exception as err:
             logger.exception(err)
             raise err
         finally:
             self._remove_file_handler()
+
+        return self.result

edu_rdm_integration/export_data/export_manger.py

@@ -3,6 +3,7 @@ from pathlib import (
 )
 from typing import (
     Iterable,
+    Optional,
 )
 
 from django.conf import (
@@ -38,6 +39,7 @@ from edu_rdm_integration.export_data.base.requests import (
     RegionalDataMartEntityRequest,
 )
 from edu_rdm_integration.export_data.consts import (
+    ATTACHMENTS_LIMIT,
     TOTAL_ATTACHMENTS_SIZE_KEY,
 )
 from edu_rdm_integration.export_data.dataclasses import (
@@ -60,10 +62,12 @@ from edu_rdm_integration.redis_cache import (
 class ExportQueueSender:
     """Класс отправки данных в очередь РВД."""
 
-    def __init__(self, data_cache: AbstractCache, queue: Queue):
+    def __init__(self, data_cache: AbstractCache, queue: Queue, limit: Optional[int] = None):
         self.queue = queue
         self.cache = data_cache
         self.queue_total_file_size = self.cache.get(TOTAL_ATTACHMENTS_SIZE_KEY) or 0
+        self.queue_is_full = False
+        self.limit = limit or ATTACHMENTS_LIMIT
 
     @staticmethod
     def get_exported_file_size() -> int:
@@ -80,22 +84,25 @@ class ExportQueueSender:
 
         return file_size or 0
 
-    @staticmethod
     def _make_stage_filter(self) -> Q:
         """Формирование фильтра для выборки подэтапов."""
         return Q(status_id=ExportingDataSubStageStatus.READY_FOR_EXPORT.key)
 
     def get_sub_stages_attachments_to_export(self):
         """Выборка готовых к экспорту подэтапов."""
-        return ExportingDataSubStage.objects.filter(
+        sub_stage_ids = ExportingDataSubStage.objects.filter(
             self._make_stage_filter()
+        ).order_by('started_at').values_list('id', flat=True)[:self.limit]
+
+        return ExportingDataSubStage.objects.filter(
+            id__in=sub_stage_ids
         ).annotate(
             attachment_id=F('exportingdatasubstageattachment__id'),
             attachment_name=F('exportingdatasubstageattachment__attachment'),
             attachment_size=F('exportingdatasubstageattachment__attachment_size'),
             operation=F('exportingdatasubstageattachment__operation'),
             entity=F('exportingdatasubstageentity__entity_id')
-        ).order_by('started_at', 'operation').values(
+        ).order_by('id', 'operation').values(
             'id',
             'attachment_id',
             'attachment_name',
@@ -120,7 +127,7 @@ class ExportQueueSender:
         )
 
         logger.info(
-            f'ExportedDataSubStage {sub_stage_id} {entity_name} added to the queue'
+             f'{LOGS_DELIMITER * 2}ExportedDataSubStage {sub_stage_id} {entity_name} added to the queue'
         )
 
         return True
@@ -129,7 +136,6 @@ class ExportQueueSender:
         """Запуск работы очереди."""
         if not self.queue_total_file_size:
             self.queue_total_file_size = self.get_exported_file_size()
-
         if self.queue_total_file_size < settings.RDM_UPLOAD_QUEUE_MAX_SIZE:
             stage_files = []
             prev_sub_stage = None
@@ -166,32 +172,18 @@ class ExportQueueSender:
                 self.queue_total_file_size,
                 timeout=settings.RDM_REDIS_CACHE_TIMEOUT_SECONDS
             )
+            self.queue_is_full = True
             logger.warning(
                 f'Total exported file size:  {self.queue_total_file_size} - queue is full!!!'
             )
 
 
-class ExportEntityQueueSender(ExportQueueSender):
-    """Класс отправки данных по конкретной сущности в очередь РВД."""
-
-    def __init__(self, data_cache: AbstractCache, queue: Queue, entity: Iterable[str]):
-        super().__init__(data_cache, queue)
-
-        self.entity = entity
-
-    def _make_stage_filter(self) -> Q:
-        """Формирование фильтра для выборки подэтапов."""
-        return Q(
-            status_id=ExportingDataSubStageStatus.READY_FOR_EXPORT.key,
-            exportingdatasubstageentity__entity_id__in=self.entity
-        )
-
-
 class WorkerSender:
     """Непосредственная отправка файлов."""
 
     def __init__(self, queue: Queue):
         self.queue = queue
+        self.entities = set()
 
     def send_files(self):
         """Отправка файлов."""
@@ -216,7 +208,8 @@ class WorkerSender:
                 try:
                     file_data = file_path.open('rb').read()
                 except (OSError, IOError, FileNotFoundError) as error:
-                    logger.error(f'Ошибка чтения файла {file_path} - {str(error)} ')
+                    logger.exception(f'Ошибка чтения файла {file_path} - {str(error)} ')
+                    status = ExportingDataSubStageStatus.FAILED.key
                     continue
 
                 request = RegionalDataMartEntityRequest(
@@ -260,17 +253,17 @@ class WorkerSender:
                     logger.info(
                         f'Response with {result.response.status_code} code and content {result.response.text}'
                     )
+                    self.entities.add(entity_key)
 
             # Сохраняем информацию об отправке файлов и убираем подэтап с файлами из очереди
-            if result_to_save:
-                with atomic():
-                    ExportingDataSubStageUploaderClientLog.objects.bulk_create(result_to_save)
-                    sub_stage.status_id = status
-                    sub_stage.save()
+            with atomic():
+                ExportingDataSubStageUploaderClientLog.objects.bulk_create(result_to_save)
+                sub_stage.status_id = status
+                sub_stage.save()
 
             self.queue.delete_from_queue(sub_stage_id=sub_stage_id, entity_name=entity_key)
             logger.info(
-                f'ExportedDataSubStage {sub_stage_id} {entity_key} sended from the queue'
+                f'{LOGS_DELIMITER * 3}ExportedDataSubStage {sub_stage_id} {entity_key} sended from the queue'
             )
 
     def run(self):

edu_rdm_integration/tasks.py

@@ -56,7 +56,7 @@ from edu_rdm_integration.enums import (
 )
 from edu_rdm_integration.export_data.export import (
     ExportLatestEntitiesData,
-    UploadEntitiesData,
+    UploadData,
 )
 from edu_rdm_integration.export_data.helpers import (
     set_failed_status_suspended_exporting_data_stages,
@@ -74,7 +74,6 @@ from edu_rdm_integration.models import (
     ExportingDataSubStageUploaderClientLog,
     RegionalDataMartEntityEnum,
     TransferredEntity,
-    UploadDataCommand,
 )
 from edu_rdm_integration.storages import (
     RegionalDataMartEntityStorage,
@@ -360,31 +359,23 @@ class UploadDataAsyncTask(UniquePeriodicAsyncTask):
         """Выполнение."""
         super().process(*args, **kwargs)
 
-        entity_ids_for_export = list(
-            TransferredEntity.objects.filter(export_enabled=True).values_list('entity_id', flat=True)
-        )
         queue = RdmRedisSubStageAttachmentQueue()
-        task_id = RunningTask.objects.filter(
-            pk=self.request.id,
-        ).values_list('pk', flat=True).first()
+        upload_data = UploadData(
+            data_cache=cache,
+            queue=queue,
+        )
+
+        upload_result = upload_data.upload_data()
 
-        if task_id:
-            for entity_id in entity_ids_for_export:
-                upload_data_command = UploadDataCommand.objects.create(
-                    entity_id=entity_id,
-                    task_id=task_id,
-                    type=CommandType.AUTO,
-                )
-                if upload_data_command:
-                    upload_data = UploadEntitiesData(
-                        entities=[upload_data_command.entity_id],
-                        data_cache=cache,
-                        queue=queue,
-                        task_id=task_id,
-                        command_id=upload_data_command.id,
-                    )
-                    upload_data.upload_data()
-                    save_command_log_link(upload_data_command, settings.RDM_UPLOAD_LOG_DIR)
+        task_result = {
+            'Общий объем отправленных файлов': f"{upload_result['total_file_size']}",
+            'Очередь отправки переполнена': 'Да' if upload_result['queue_is_full'] else 'Нет',
+            'Сущности, отправленные в витрину': upload_result['uploaded_entities']
+        }
+
+        self.set_progress(
+            values=task_result
+        )
 
 
 celery_app = celery.app.app_or_default()

{edu_rdm_integration-3.5.2.dist-info → edu_rdm_integration-3.5.4.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: edu-rdm-integration
-Version: 3.5.2
+Version: 3.5.4
 Summary: Интеграция с Региональной витриной данных
 Home-page: 
 Download-URL: 
@@ -301,6 +301,27 @@ Requires-Dist: uploader-client <1,>=0.2.1
 ### Удалено
 
 
+## [3.5.4] - 2025-02-07
+
+Внесено исправление в BaseIgnoreLogMixin, в части работы метода _exclude_logs.
+
+### Изменено
+
+- [EDUSCHL-23160](https://jira.bars.group/browse/EDUSCHL-23160)
+  PATCH Внесено исправление в BaseIgnoreLogMixin, в части работы метода _exclude_logs.
+
+
+## [3.5.3] - 2025-01-29
+
+Изменена периодическая задача UploadDataAsyncTask.
+
+### Изменено
+
+- [EDUSCHL-23092](https://jira.bars.group/browse/EDUSCHL-23092)
+  PATCH Изменена периодическая задача UploadDataAsyncTask - удалено формирование очереди по сущностям. Сейчас в очередь
+  попадает указанное в настройках количество файлов, готовых к отправке в витрину
+
+
 ## [3.5.2] - 2025-02-04
 
 Изменён тип поля institute_ids с django.contrib.postgres.fields.JSONField (поддержка до Django 4.0.)

{edu_rdm_integration-3.5.2.dist-info → edu_rdm_integration-3.5.4.dist-info}/RECORD

@@ -1,6 +1,6 @@
 edu_rdm_integration/__init__.py,sha256=fVCvQ7QGI_iCyAeE8dMapyY8gOM617ye5GQqAVGPlZI,72
 edu_rdm_integration/app_meta.py,sha256=v5IU69yaeLbyHF0Ln6iPN_IfizbtF3rCWrz2n71m8dU,337
-edu_rdm_integration/app_settings.py,sha256=jcMh5WC2_TbwGNZ2FsdlQX2u9MWEZ5XPklKSqOAZzTo,2813
+edu_rdm_integration/app_settings.py,sha256=FnTCsAM7lvd05q4YQqZnCS31plnluEJmUtIwJ6zunoM,2996
 edu_rdm_integration/apps.py,sha256=iDNitsYHE1-djp2NCW4HKnFHNoDSg8Fzc8Tobe26Qzw,3571
 edu_rdm_integration/base.py,sha256=_G0qPTAXe6bXfgDHNiZMSsYt3sMuUhLKnHuQCWSFttU,1341
 edu_rdm_integration/consts.py,sha256=Qt52SOCQ-3wOet-_6inJih_W9nToORKXbkxb3jVSjEo,1079
@@ -12,7 +12,7 @@ edu_rdm_integration/models.py,sha256=e1n-1UuG0Okhx6NSxD-m3iSlvStDmBdJs-4HiBa8aTQ
 edu_rdm_integration/redis_cache.py,sha256=SP_rcL5t6PTVLOnEYn_NTX0Z666VdZT4By2pyED24Z4,1537
 edu_rdm_integration/signals.py,sha256=3eRlpkDcFCF6TN80-QM8yBYLcyozzcmoPjz6r4_ApWg,73
 edu_rdm_integration/storages.py,sha256=G4Q4tIyJdEyb9ka551PADCFIm66bpsJe9VBRcvQhLMI,6745
-edu_rdm_integration/tasks.py,sha256=TP1oc32xahTQsX_vhWxYaCxDJdw9-iN2L1Lukins3K0,16024
+edu_rdm_integration/tasks.py,sha256=ie9fduL1QMR_ndFXIruY4VXdTnStqiV2vd6iNILFxQw,15540
 edu_rdm_integration/typing.py,sha256=2asD8biX0l_DVqJSU4y19-zzEBJMk967PUj3UhzICzs,785
 edu_rdm_integration/utils.py,sha256=NNRfblJ9QoX07qUa2kfKG0jM3fHJ3oaDvG2zwj0As70,12120
 edu_rdm_integration/adapters/__init__.py,sha256=cU0swn4Ny5ZQz5buWRcWsT1mpWuUFJaUlHf2l7TtEBo,83
@@ -49,7 +49,7 @@ edu_rdm_integration/collect_data/base/caches.py,sha256=_Ja0q3hPSE_Mq-MCZN-1jr0Wv
 edu_rdm_integration/collect_data/base/functions.py,sha256=E1vYmC8F8NM5cBB5SGQEpluHrMynpjf3Ek3s0HsbxwY,2998
 edu_rdm_integration/collect_data/base/helpers.py,sha256=MsHEzkUl1KVMIxGfDkhcItXLxQNuNNOhN5tzTcPm0NQ,982
 edu_rdm_integration/collect_data/base/managers.py,sha256=hza8kugR9hbLK3LU8DBfF61ul7mPPl5pPm530QMjj0k,6143
-edu_rdm_integration/collect_data/base/mixins.py,sha256=K3HOWaxAE3Myzh8CgTm-ZKE7_hbW9p6IcCnvtbDzlM4,8482
+edu_rdm_integration/collect_data/base/mixins.py,sha256=kwZ77dc5dCmHVGCN6JMslpyfXfEEoks7vmXH-CzezJc,8538
 edu_rdm_integration/collect_data/base/runners.py,sha256=Mf5Lux6IWWqKd798v149OjqfBWiDtigeFd701HSRBGg,1565
 edu_rdm_integration/collect_data/calculated/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 edu_rdm_integration/collect_data/calculated/strategies.py,sha256=nppmheuzhF8H7JcfEhEZCmHI0eeRj8BbrLGNF0nsoa0,7619
@@ -87,10 +87,10 @@ edu_rdm_integration/enum_register/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeR
 edu_rdm_integration/enum_register/mixins.py,sha256=lry5JDGIleBCcOqsfX0sVOqwnnSV2Jlb4T2heD-xg54,4621
 edu_rdm_integration/enum_register/register.py,sha256=jb16O7TZDGVRo5NwWqyY_rcbBxOncYeOoSq6UAbTf48,2208
 edu_rdm_integration/export_data/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-edu_rdm_integration/export_data/consts.py,sha256=rmJ_18wHCE54j0VABxfo7Zu01EPFYSjYrj5L79hVW2Y,333
+edu_rdm_integration/export_data/consts.py,sha256=ZEi1kXMs-54KFKxkyGIQVwZ4d8OrOF_vLFQIjjWdSPQ,441
 edu_rdm_integration/export_data/dataclasses.py,sha256=IhftRopP4lS-m3ygdBU5Bz0HF71VSBP4JQ6-8VIVgtY,260
-edu_rdm_integration/export_data/export.py,sha256=h2WjbgkrFpp7pA5XmDgKjhn_n-C7vgULCgN3dT1E1S4,18474
-edu_rdm_integration/export_data/export_manger.py,sha256=xAokqaYi00GA7Otsu9pfwqMRQXtYReoX6g1BSjAJtnU,10937
+edu_rdm_integration/export_data/export.py,sha256=WSnli8-ZBtZw4b2_8qnmPTgKW52XMdZ9RlJ4eo25xyQ,19071
+edu_rdm_integration/export_data/export_manger.py,sha256=Q1O1lO8kF0s4jzQsMj-qWcFe14_YPM0AXXWywXWj-z8,10776
 edu_rdm_integration/export_data/generators.py,sha256=UIoX49rQnUwwC9PL17te7Rb4WRDe_5ShYQ1Rygf36NQ,4010
 edu_rdm_integration/export_data/helpers.py,sha256=JKL4MC0IuoJv72NNnL8hG7HLy7kNHOXFp1uLCKF15AM,2900
 edu_rdm_integration/export_data/queue.py,sha256=GlwRwhMdv38OxmtFpFD5-Pt79gMe3IcLeWFQCLXTcik,6128
@@ -178,9 +178,9 @@ edu_rdm_integration/uploader_log/enums.py,sha256=rgSO3BL2rh2xpfm0Pt4waQW8fB1VMJL
 edu_rdm_integration/uploader_log/managers.py,sha256=y5wTSMzF9hpOpIU_A7nIafL_LBU3QEie6LAYWoB-pBQ,3203
 edu_rdm_integration/uploader_log/ui.py,sha256=YM9Buqp2wxE95Wf5gvAATBzuYzDOossK1sEmvFk07cI,2110
 edu_rdm_integration/uploader_log/templates/ui-js/object-grid-buttons.js,sha256=2xyGe0wdVokM0RhpzRzcRvJPBkBmPe3SlZry4oP4Nzs,6201
-edu_rdm_integration-3.5.2.dist-info/LICENSE,sha256=uw43Gjjj-1vXWCItfSrNDpbejnOwZMrNerUh8oWbq8Q,3458
-edu_rdm_integration-3.5.2.dist-info/METADATA,sha256=Y8n72AFmj2fGwdui8iYNWtQcdb2iZuf7iWlxCdNT4wc,78751
-edu_rdm_integration-3.5.2.dist-info/WHEEL,sha256=yQN5g4mg4AybRjkgi-9yy4iQEFibGQmlz78Pik5Or-A,92
-edu_rdm_integration-3.5.2.dist-info/namespace_packages.txt,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
-edu_rdm_integration-3.5.2.dist-info/top_level.txt,sha256=nRJV0O14UtNE-jGIYG03sohgFnZClvf57H5m6VBXe9Y,20
-edu_rdm_integration-3.5.2.dist-info/RECORD,,
+edu_rdm_integration-3.5.4.dist-info/LICENSE,sha256=uw43Gjjj-1vXWCItfSrNDpbejnOwZMrNerUh8oWbq8Q,3458
+edu_rdm_integration-3.5.4.dist-info/METADATA,sha256=NkUahYegND2vl7RrspU-lhHOIL0dsye-AEdszKoQCdY,79645
+edu_rdm_integration-3.5.4.dist-info/WHEEL,sha256=yQN5g4mg4AybRjkgi-9yy4iQEFibGQmlz78Pik5Or-A,92
+edu_rdm_integration-3.5.4.dist-info/namespace_packages.txt,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
+edu_rdm_integration-3.5.4.dist-info/top_level.txt,sha256=nRJV0O14UtNE-jGIYG03sohgFnZClvf57H5m6VBXe9Y,20
+edu_rdm_integration-3.5.4.dist-info/RECORD,,