edu-rdm-integration 3.5.5__py3-none-any.whl → 3.5.7__py3-none-any.whl

edu_rdm_integration/app_settings.py

@@ -59,4 +59,6 @@ RDM_UPLOAD_DATA_TASK_HOUR = '*/2'
 RDM_UPLOAD_DATA_TASK_DAY_OF_WEEK = '*'
 RDM_UPLOAD_DATA_TASK_LOCK_EXPIRE_SECONDS = 60 * 60 * 2
 # Количество подэтапов для обработки в периодической задаче отправки данных
-RDM_UPLOAD_DATA_TASK_EXPORT_STAGES = 500
+RDM_UPLOAD_DATA_TASK_EXPORT_STAGES = 500
+
+RDM_RESPONSE_FILE_STATUS = 'success'

edu_rdm_integration/apps.py

@@ -38,16 +38,18 @@ class EduRDMIntegrationConfig(AppConfig):
         )
         from uploader_client.contrib.rdm.interfaces.configurations import (
             RegionalDataMartUploaderConfig,
+            RegionalDataMartEmulationUploaderConfig,
         )
 
         if settings.RDM_UPLOADER_CLIENT_ENABLE_REQUEST_EMULATION:
             uploader_client.set_config(
-                RegionalDataMartUploaderConfig(
+                RegionalDataMartEmulationUploaderConfig(
                     interface='uploader_client.contrib.rdm.interfaces.rest.OpenAPIInterfaceEmulation',
                     url=settings.RDM_UPLOADER_CLIENT_URL,
                     datamart_name=settings.RDM_UPLOADER_CLIENT_DATAMART_NAME,
                     timeout=1,
                     request_retries=1,
+                    file_status=settings.RDM_RESPONSE_FILE_STATUS
                 )
             )
         elif settings.RDM_UPLOADER_CLIENT_USE_PROXY_API:

edu_rdm_integration/consts.py

@@ -34,3 +34,4 @@ TASK_QUEUE_NAME = 'RDM'
 # Кортеж операций для обновления данных
 UPDATED_OPERATIONS = (EntityLogOperation.CREATE, EntityLogOperation.UPDATE)
 ALL_OPERATIONS = UPDATED_OPERATIONS + (EntityLogOperation.DELETE, )
+

edu_rdm_integration/export_data/export.py

@@ -64,10 +64,10 @@ from edu_rdm_integration.base import (
 from edu_rdm_integration.consts import (
     REGIONAL_DATA_MART_INTEGRATION_EXPORTING_DATA,
 )
-from edu_rdm_integration.enums import (
-    FileUploadStatusEnum,
+from edu_rdm_integration.export_data.consts import (
+    TOTAL_ATTACHMENTS_SIZE_KEY,
 )
-from edu_rdm_integration.export_data.export_manger import (
+from edu_rdm_integration.export_data.export_manager import (
     ExportQueueSender,
     WorkerSender,
 )
@@ -294,14 +294,6 @@ class BaseExportLatestEntitiesData(BaseExportEntitiesData):
                 modified__gt=F('exporting_sub_stage__ended_at'),
                 modified__lt=last_finished_export_data,
             ),
-            # Связь лога подэтапа выгрузки с логом загрузки данных (ExportingDataSubStageUploaderClientLog)
-            # имеет статус FileUploadStatusEnum.ERROR:
-            model.objects.filter(
-                exporting_sub_stage__ended_at__gt=now - timedelta(days=365),
-                exporting_sub_stage__exportingdatasubstageuploaderclientlog__created__lt=last_finished_export_data,
-                exporting_sub_stage__exportingdatasubstageuploaderclientlog__file_upload_status=(
-                    FileUploadStatusEnum.ERROR),
-            ),
         )
 
         for queryset in querysets_to_update:
@@ -419,16 +411,18 @@ class UploadData(BaseOperationData):
             caches,
         )
         from uploader_client.contrib.rdm.interfaces.configurations import (
+            RegionalDataMartEmulationUploaderConfig,
             RegionalDataMartUploaderConfig,
         )
         if settings.RDM_UPLOADER_CLIENT_ENABLE_REQUEST_EMULATION:
             uploader_client.set_config(
-                RegionalDataMartUploaderConfig(
+                RegionalDataMartEmulationUploaderConfig(
                     interface='uploader_client.contrib.rdm.interfaces.rest.OpenAPIInterfaceEmulation',
                     url=settings.RDM_UPLOADER_CLIENT_URL,
                     datamart_name=settings.RDM_UPLOADER_CLIENT_DATAMART_NAME,
                     timeout=1,
                     request_retries=1,
+                    file_status=settings.RDM_RESPONSE_FILE_STATUS
                 )
             )
         elif settings.RDM_UPLOADER_CLIENT_USE_PROXY_API:
@@ -457,6 +451,19 @@ class UploadData(BaseOperationData):
                 )
             )
 
+    def update_total_queue_size_in_cache(self, received_files_size: int):
+        """Обновление размера файлов в кеш."""
+        with self.data_cache.lock(f'{TOTAL_ATTACHMENTS_SIZE_KEY}:lock', timeout=300):
+            queue_total_file_size = self.data_cache.get(TOTAL_ATTACHMENTS_SIZE_KEY) or 0
+            if queue_total_file_size:
+                queue_total_file_size -= received_files_size
+                if queue_total_file_size > 0:
+                    self.data_cache.set(
+                        TOTAL_ATTACHMENTS_SIZE_KEY,
+                        queue_total_file_size,
+                        timeout=settings.RDM_REDIS_CACHE_TIMEOUT_SECONDS
+                    )
+
     def upload_data(self, *args, **kwargs):
         """
         Запускает отправку данных в витрину.
@@ -476,6 +483,9 @@ class UploadData(BaseOperationData):
                 if sender.entities:
                     self.result['uploaded_entities'] = ','.join(sender.entities)
 
+                if sender.received_file_size:
+                    self.update_total_queue_size_in_cache(sender.received_file_size)
+
         except Exception as err:
             logger.exception(err)
             raise err

edu_rdm_integration/export_data/{export_manger.py → export_manager.py}

@@ -2,6 +2,8 @@ from pathlib import (
     Path,
 )
 from typing import (
+    TYPE_CHECKING,
+    Any,
     Iterable,
     Optional,
 )
@@ -48,11 +50,17 @@ from edu_rdm_integration.export_data.dataclasses import (
 from edu_rdm_integration.export_data.queue import (
     Queue,
 )
+from edu_rdm_integration.helpers import (
+    FAILED_STATUSES,
+    UploadStatusHelper,
+)
 from edu_rdm_integration.models import (
+    DataMartRequestStatus,
     ExportingDataSubStage,
     ExportingDataSubStageAttachment,
     ExportingDataSubStageStatus,
     ExportingDataSubStageUploaderClientLog,
+    UploadStatusRequestLog,
 )
 from edu_rdm_integration.redis_cache import (
     AbstractCache,
@@ -184,6 +192,31 @@ class WorkerSender:
     def __init__(self, queue: Queue):
         self.queue = queue
         self.entities = set()
+        self.received_file_size = 0
+
+    def get_file_upload_status(self, request_id: str) -> tuple[Optional[int], Optional[DataMartRequestStatus], 'Entry']:
+        """Возвращает статус файла в витрине по запросу."""
+        file_upload_status = None
+
+        response, log_entry = UploadStatusHelper.send_upload_status_request(request_id)
+        request_status = None
+
+        if response:
+            request_status = DataMartRequestStatus.get_values_to_enum_data().get(response.get('code'))
+
+            if not request_status:
+                logger.error(
+                    'Не удалось определить статус загрузки данных в витрину. Идентификатор загрузки: '
+                    f'{request_id}, данные ответа: {response}',
+                )
+
+        if request_status in FAILED_STATUSES:
+            file_upload_status = FileUploadStatusEnum.ERROR
+
+        elif request_status == DataMartRequestStatus.SUCCESSFULLY_PROCESSED:
+            file_upload_status = FileUploadStatusEnum.FINISHED
+
+        return file_upload_status, request_status, log_entry
 
     def send_files(self):
         """Отправка файлов."""
@@ -194,6 +227,7 @@ class WorkerSender:
                 return
 
             result_to_save = []
+            upload_status_requests_to_save = []
             errors = []
             status = ExportingDataSubStageStatus.FINISHED.key
             for file in upload_files:
@@ -241,6 +275,25 @@ class WorkerSender:
                     file_upload_status=file_upload_status,
                     is_emulation=settings.RDM_UPLOADER_CLIENT_ENABLE_REQUEST_EMULATION,
                 )
+
+                if request_id:
+                    updated_file_upload_status, request_status, log_entry = self.get_file_upload_status(request_id)
+                    upload_status_requests_to_save.append(
+                        UploadStatusRequestLog(
+                            upload=sub_stage_uploader_client_log,
+                            entry=log_entry,
+                            request_status_id=getattr(request_status, 'key', None),
+                        )
+                    )
+                    if updated_file_upload_status and updated_file_upload_status != file_upload_status:
+                        sub_stage_uploader_client_log.file_upload_status = updated_file_upload_status
+
+                    if updated_file_upload_status == FileUploadStatusEnum.FINISHED:
+                        self.received_file_size += file.attachment_size
+
+                    if updated_file_upload_status == FileUploadStatusEnum.ERROR:
+                        status = ExportingDataSubStageStatus.PROCESS_ERROR.key
+
                 result_to_save.append(sub_stage_uploader_client_log)
                 if result.error:
                     logger.warning(
@@ -258,6 +311,7 @@ class WorkerSender:
             # Сохраняем информацию об отправке файлов и убираем подэтап с файлами из очереди
             with atomic():
                 ExportingDataSubStageUploaderClientLog.objects.bulk_create(result_to_save)
+                UploadStatusRequestLog.objects.bulk_create(upload_status_requests_to_save)
                 sub_stage.status_id = status
                 sub_stage.save()
 

edu_rdm_integration/helpers.py

@@ -56,6 +56,7 @@ from edu_rdm_integration.models import (
     DataMartRequestStatus,
     ExportingDataStage,
     ExportingDataStageStatus,
+    ExportingDataSubStageStatus,
     ExportingDataSubStageUploaderClientLog,
     UploadDataCommand,
     UploadStatusRequestLog,
@@ -82,6 +83,13 @@ if TYPE_CHECKING:
     )
 
 
+FAILED_STATUSES = {
+    DataMartRequestStatus.FAILED_PROCESSING,
+    DataMartRequestStatus.REQUEST_ID_NOT_FOUND,
+    DataMartRequestStatus.FLC_ERROR
+}
+
+
 class UploadStatusHelper:
     """Хелпер проверки статуса загрузки данных в витрину."""
 
@@ -160,8 +168,10 @@ class UploadStatusHelper:
                 request_status_id=getattr(request_status, 'key', None),
             )
 
-            if request_status in {DataMartRequestStatus.FAILED_PROCESSING, DataMartRequestStatus.REQUEST_ID_NOT_FOUND}:
+            if request_status in FAILED_STATUSES:
                 upload.file_upload_status = FileUploadStatusEnum.ERROR
+                upload.sub_stage.status_id = ExportingDataSubStageStatus.PROCESS_ERROR.key
+                upload.sub_stage.save()
 
             elif request_status == DataMartRequestStatus.SUCCESSFULLY_PROCESSED:
                 upload.file_upload_status = FileUploadStatusEnum.FINISHED

edu_rdm_integration/migrations/0015_set_exporting_sub_stage_status.py

@@ -0,0 +1,25 @@
+# Generated by Django 3.2.24 on 2025-02-14 15:49
+
+from django.db import (
+    migrations,
+)
+
+
+def add_exporting_data_sub_stage_status(apps, schema_editor):
+    """Добавление нового статуса этапу выгрузки данных."""
+    ExportingDataSubStageStatus = apps.get_model('edu_rdm_integration', 'ExportingDataSubStageStatus')  # noqa: N806
+    ExportingDataSubStageStatus.objects.get_or_create(
+        key='PROCESS_ERROR',
+        title='Ошибка обработки витриной'
+    )
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('edu_rdm_integration', '0014_uploaddatacommand'),
+    ]
+
+    operations = [
+        migrations.RunPython(add_exporting_data_sub_stage_status, reverse_code=migrations.RunPython.noop),
+    ]

edu_rdm_integration/models.py

@@ -18,13 +18,13 @@ from django.db.models import (
     DateTimeField,
     FileField,
     ForeignKey,
+    JSONField,
     Manager,
     OneToOneField,
     PositiveIntegerField,
     PositiveSmallIntegerField,
     SmallIntegerField,
     UUIDField,
-    JSONField,
 )
 from django.utils import (
     timezone,
@@ -370,6 +370,9 @@ class ExportingDataSubStageStatus(TitledModelEnum):
     READY_FOR_EXPORT = ModelEnumValue(
         title='Готов к выгрузке',
     )
+    PROCESS_ERROR = ModelEnumValue(
+        title='Ошибка обработки витриной'
+    )
 
     class Meta:
         db_table = 'rdm_exporting_data_sub_stage_status'

{edu_rdm_integration-3.5.5.dist-info → edu_rdm_integration-3.5.7.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: edu-rdm-integration
-Version: 3.5.5
+Version: 3.5.7
 Summary: Интеграция с Региональной витриной данных
 Home-page: 
 Download-URL: 
@@ -31,7 +31,7 @@ Requires-Dist: educommon <4,>=3.11.0
 Requires-Dist: function-tools <1,>=0.9.0
 Requires-Dist: m3-db-utils <1,>=0.3.10
 Requires-Dist: m3-django-compat <2,>=1.10.2
-Requires-Dist: uploader-client <1,>=0.2.1
+Requires-Dist: uploader-client <1,>=0.2.3
 
 # Проект "Интеграция с Региональной витриной данных (РВД)"
 
@@ -300,6 +300,28 @@ Requires-Dist: uploader-client <1,>=0.2.1
 
 ### Удалено
 
+## [3.5.7] - 2025-03-03
+
+Испарвлена ошибка импорта в export_manager.
+
+### Исправлено
+
+- [EDUSCHL-22403](https://jira.bars.group/browse/EDUSCHL-22403)
+  PATCH Испарвлена ошибка импорта в export_manager.
+
+
+## [3.5.6] - 2025-02-20
+
+Доработана отправка файлов в витрину - добавлена проверка статусов файла и новый статус `Ошибка обработки витриной` для
+ExportingDataSubStageStatus. Доработана задача проверки статусов  UploadStatusHelper.
+
+### Исправлено
+
+- [EDUSCHL-22403](https://jira.bars.group/browse/EDUSCHL-22403)
+  PATCH Добавлен новый статус Ошибка обработки витриной для ExportingDataSubStageStatus. Добавлена проверка 
+  статусов сразу после отправки файлов WorkerSender. Скорректирован статус UploadStatusHelper - при получении статусов 
+  ошибки обработки витриной подэтап помечается как PROCESS_ERROR
+
 
 ## [3.5.5] - 2025-02-11
 

{edu_rdm_integration-3.5.5.dist-info → edu_rdm_integration-3.5.7.dist-info}/RECORD

@@ -1,14 +1,14 @@
 edu_rdm_integration/__init__.py,sha256=fVCvQ7QGI_iCyAeE8dMapyY8gOM617ye5GQqAVGPlZI,72
 edu_rdm_integration/app_meta.py,sha256=v5IU69yaeLbyHF0Ln6iPN_IfizbtF3rCWrz2n71m8dU,337
-edu_rdm_integration/app_settings.py,sha256=FnTCsAM7lvd05q4YQqZnCS31plnluEJmUtIwJ6zunoM,2996
-edu_rdm_integration/apps.py,sha256=iDNitsYHE1-djp2NCW4HKnFHNoDSg8Fzc8Tobe26Qzw,3571
+edu_rdm_integration/app_settings.py,sha256=f7Obftma3sQSXZmQRkfas93VFnJi3eTGVE_JixzxAnQ,3035
+edu_rdm_integration/apps.py,sha256=Dl1og2yZcRyJmqrifUNIjLZ9Us2jw2mgAP1_q2fg29A,3699
 edu_rdm_integration/base.py,sha256=_G0qPTAXe6bXfgDHNiZMSsYt3sMuUhLKnHuQCWSFttU,1341
-edu_rdm_integration/consts.py,sha256=Qt52SOCQ-3wOet-_6inJih_W9nToORKXbkxb3jVSjEo,1079
+edu_rdm_integration/consts.py,sha256=nL8o-w73KjGM4r_--Nasffmo_DJx9HsgrPHSNtq-y-g,1080
 edu_rdm_integration/entities.py,sha256=mhVeB88A-VD5IAzZCNeI1qnkvNoZ8LPiLBdqk1yA3Jc,14541
 edu_rdm_integration/enums.py,sha256=T3Mu5D-CbKO3BSg16MPPnIPlcc_YGLYR-ThS8dzl9gg,4246
-edu_rdm_integration/helpers.py,sha256=pLWkIUZK0M5FZZrTAfe80bV-VZIpBgwxnXBAmYFfFw0,14561
+edu_rdm_integration/helpers.py,sha256=wr4ddI9LNsmcwdZMEUYT070YnM49ixbAQ0-tBh4gp08,14808
 edu_rdm_integration/mapping.py,sha256=1B6TsC4Os9wiM8L8BChnCNv_iWqjeWu3bdDsqKVsId0,616
-edu_rdm_integration/models.py,sha256=e1n-1UuG0Okhx6NSxD-m3iSlvStDmBdJs-4HiBa8aTQ,32985
+edu_rdm_integration/models.py,sha256=feAbpH1ji4MOHpYtzDfrumMYmXlz1ye-IDUFJSI7Nq0,33092
 edu_rdm_integration/redis_cache.py,sha256=SP_rcL5t6PTVLOnEYn_NTX0Z666VdZT4By2pyED24Z4,1537
 edu_rdm_integration/signals.py,sha256=3eRlpkDcFCF6TN80-QM8yBYLcyozzcmoPjz6r4_ApWg,73
 edu_rdm_integration/storages.py,sha256=G4Q4tIyJdEyb9ka551PADCFIm66bpsJe9VBRcvQhLMI,6745
@@ -89,8 +89,8 @@ edu_rdm_integration/enum_register/register.py,sha256=jb16O7TZDGVRo5NwWqyY_rcbBxO
 edu_rdm_integration/export_data/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 edu_rdm_integration/export_data/consts.py,sha256=ZEi1kXMs-54KFKxkyGIQVwZ4d8OrOF_vLFQIjjWdSPQ,441
 edu_rdm_integration/export_data/dataclasses.py,sha256=IhftRopP4lS-m3ygdBU5Bz0HF71VSBP4JQ6-8VIVgtY,260
-edu_rdm_integration/export_data/export.py,sha256=WSnli8-ZBtZw4b2_8qnmPTgKW52XMdZ9RlJ4eo25xyQ,19071
-edu_rdm_integration/export_data/export_manger.py,sha256=Q1O1lO8kF0s4jzQsMj-qWcFe14_YPM0AXXWywXWj-z8,10776
+edu_rdm_integration/export_data/export.py,sha256=qKvFRTcy2ZgaBMn6mEi_OsueNXZ7j_v2neu2Lu99MEs,19447
+edu_rdm_integration/export_data/export_manager.py,sha256=ySAbrIiVo690HXGsEC_nxhF5FHCRl18rm5E4Ytr50VU,13285
 edu_rdm_integration/export_data/generators.py,sha256=UIoX49rQnUwwC9PL17te7Rb4WRDe_5ShYQ1Rygf36NQ,4010
 edu_rdm_integration/export_data/helpers.py,sha256=JKL4MC0IuoJv72NNnL8hG7HLy7kNHOXFp1uLCKF15AM,2900
 edu_rdm_integration/export_data/queue.py,sha256=GlwRwhMdv38OxmtFpFD5-Pt79gMe3IcLeWFQCLXTcik,6128
@@ -166,6 +166,7 @@ edu_rdm_integration/migrations/0011_exportingdatasubstageentity.py,sha256=2BfIif
 edu_rdm_integration/migrations/0012_exportingdatasubstageattachment_attachment_size.py,sha256=y_JQO69k9pEfrJyimaRiAOBmhaJmssIyepCGd-Sy9hs,511
 edu_rdm_integration/migrations/0013_set_attachment_size.py,sha256=Gol8T137gdaCTSkJ2e4as5x4gfqeouZmgWgkOl7zxCQ,2048
 edu_rdm_integration/migrations/0014_uploaddatacommand.py,sha256=Hh0vKKiGgKOvY1kBAcmway4dSYUXwVArHAc9YrsjCIU,2079
+edu_rdm_integration/migrations/0015_set_exporting_sub_stage_status.py,sha256=zVe2baNq8JYzMPRmtpAwplmgKHOP3lwMKHLo_yRz0QE,790
 edu_rdm_integration/migrations/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 edu_rdm_integration/registry/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 edu_rdm_integration/registry/actions.py,sha256=jNEq1RhSGtxdYOIKcTtqT1LPUfKvOIFgAQjP3q0Lv1s,5382
@@ -178,9 +179,9 @@ edu_rdm_integration/uploader_log/enums.py,sha256=rgSO3BL2rh2xpfm0Pt4waQW8fB1VMJL
 edu_rdm_integration/uploader_log/managers.py,sha256=OFdToWV8qhdfeGNpd-UWAmSEISzixmVQ6LF75EW7gzA,3248
 edu_rdm_integration/uploader_log/ui.py,sha256=YM9Buqp2wxE95Wf5gvAATBzuYzDOossK1sEmvFk07cI,2110
 edu_rdm_integration/uploader_log/templates/ui-js/object-grid-buttons.js,sha256=2xyGe0wdVokM0RhpzRzcRvJPBkBmPe3SlZry4oP4Nzs,6201
-edu_rdm_integration-3.5.5.dist-info/LICENSE,sha256=uw43Gjjj-1vXWCItfSrNDpbejnOwZMrNerUh8oWbq8Q,3458
-edu_rdm_integration-3.5.5.dist-info/METADATA,sha256=5snRB_tVTAg1nQ1xhcnIHbdZ4cA3P77ElM-4EU7QgxM,80191
-edu_rdm_integration-3.5.5.dist-info/WHEEL,sha256=yQN5g4mg4AybRjkgi-9yy4iQEFibGQmlz78Pik5Or-A,92
-edu_rdm_integration-3.5.5.dist-info/namespace_packages.txt,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
-edu_rdm_integration-3.5.5.dist-info/top_level.txt,sha256=nRJV0O14UtNE-jGIYG03sohgFnZClvf57H5m6VBXe9Y,20
-edu_rdm_integration-3.5.5.dist-info/RECORD,,
+edu_rdm_integration-3.5.7.dist-info/LICENSE,sha256=uw43Gjjj-1vXWCItfSrNDpbejnOwZMrNerUh8oWbq8Q,3458
+edu_rdm_integration-3.5.7.dist-info/METADATA,sha256=vulkiuukw61qOjoU9u84Met068PnWwwBndY7-k-zw9g,81379
+edu_rdm_integration-3.5.7.dist-info/WHEEL,sha256=yQN5g4mg4AybRjkgi-9yy4iQEFibGQmlz78Pik5Or-A,92
+edu_rdm_integration-3.5.7.dist-info/namespace_packages.txt,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
+edu_rdm_integration-3.5.7.dist-info/top_level.txt,sha256=nRJV0O14UtNE-jGIYG03sohgFnZClvf57H5m6VBXe9Y,20
+edu_rdm_integration-3.5.7.dist-info/RECORD,,