edu-rdm-integration 3.5.5__tar.gz → 3.5.6__tar.gz

{edu-rdm-integration-3.5.5 → edu-rdm-integration-3.5.6}/CHANGELOG.md

@@ -28,6 +28,18 @@
 
 ### Удалено
 
+## [3.5.6] - 2025-02-20
+
+Доработана отправка файлов в витрину - добавлена проверка статусов файла и новый статус `Ошибка обработки витриной` для
+ExportingDataSubStageStatus. Доработана задача проверки статусов  UploadStatusHelper.
+
+### Исправлено
+
+- [EDUSCHL-22403](https://jira.bars.group/browse/EDUSCHL-22403)
+  PATCH Добавлен новый статус Ошибка обработки витриной для ExportingDataSubStageStatus. Добавлена проверка 
+  статусов сразу после отправки файлов WorkerSender. Скорректирован статус UploadStatusHelper - при получении статусов 
+  ошибки обработки витриной подэтап помечается как PROCESS_ERROR
+
 
 ## [3.5.5] - 2025-02-11
 

{edu-rdm-integration-3.5.5 → edu-rdm-integration-3.5.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: edu-rdm-integration
-Version: 3.5.5
+Version: 3.5.6
 Summary: Интеграция с Региональной витриной данных
 Home-page: 
 Download-URL: 
@@ -31,7 +31,7 @@ Requires-Dist: educommon<4,>=3.11.0
 Requires-Dist: function-tools<1,>=0.9.0
 Requires-Dist: m3-db-utils<1,>=0.3.10
 Requires-Dist: m3-django-compat<2,>=1.10.2
-Requires-Dist: uploader-client<1,>=0.2.1
+Requires-Dist: uploader-client<1,>=0.2.3
 
 # Проект "Интеграция с Региональной витриной данных (РВД)"
 
@@ -300,6 +300,18 @@ Requires-Dist: uploader-client<1,>=0.2.1
 
 ### Удалено
 
+## [3.5.6] - 2025-02-20
+
+Доработана отправка файлов в витрину - добавлена проверка статусов файла и новый статус `Ошибка обработки витриной` для
+ExportingDataSubStageStatus. Доработана задача проверки статусов  UploadStatusHelper.
+
+### Исправлено
+
+- [EDUSCHL-22403](https://jira.bars.group/browse/EDUSCHL-22403)
+  PATCH Добавлен новый статус Ошибка обработки витриной для ExportingDataSubStageStatus. Добавлена проверка 
+  статусов сразу после отправки файлов WorkerSender. Скорректирован статус UploadStatusHelper - при получении статусов 
+  ошибки обработки витриной подэтап помечается как PROCESS_ERROR
+
 
 ## [3.5.5] - 2025-02-11
 

{edu-rdm-integration-3.5.5 → edu-rdm-integration-3.5.6}/requirements/dependencies/internal_packages.txt

@@ -2,4 +2,4 @@ educommon>=3.11.0,<4
 function-tools>=0.9.0,<1
 m3-db-utils>=0.3.10,<1
 m3-django-compat>=1.10.2,<2
-uploader-client>=0.2.1,<1
+uploader-client>=0.2.3,<1

{edu-rdm-integration-3.5.5 → edu-rdm-integration-3.5.6}/setup.py

@@ -17,7 +17,7 @@ from setuptools import (
 
 PROJECT = 'edu_rdm_integration'
 
-VERSION = '3.5.5'
+VERSION = '3.5.6'
 
 current_dir_path = Path().resolve()
 

{edu-rdm-integration-3.5.5 → edu-rdm-integration-3.5.6}/src/edu_rdm_integration/app_settings.py

@@ -59,4 +59,6 @@ RDM_UPLOAD_DATA_TASK_HOUR = '*/2'
 RDM_UPLOAD_DATA_TASK_DAY_OF_WEEK = '*'
 RDM_UPLOAD_DATA_TASK_LOCK_EXPIRE_SECONDS = 60 * 60 * 2
 # Количество подэтапов для обработки в периодической задаче отправки данных
-RDM_UPLOAD_DATA_TASK_EXPORT_STAGES = 500
+RDM_UPLOAD_DATA_TASK_EXPORT_STAGES = 500
+
+RDM_RESPONSE_FILE_STATUS = 'success'

{edu-rdm-integration-3.5.5 → edu-rdm-integration-3.5.6}/src/edu_rdm_integration/apps.py

@@ -38,16 +38,18 @@ class EduRDMIntegrationConfig(AppConfig):
         )
         from uploader_client.contrib.rdm.interfaces.configurations import (
             RegionalDataMartUploaderConfig,
+            RegionalDataMartEmulationUploaderConfig,
         )
 
         if settings.RDM_UPLOADER_CLIENT_ENABLE_REQUEST_EMULATION:
             uploader_client.set_config(
-                RegionalDataMartUploaderConfig(
+                RegionalDataMartEmulationUploaderConfig(
                     interface='uploader_client.contrib.rdm.interfaces.rest.OpenAPIInterfaceEmulation',
                     url=settings.RDM_UPLOADER_CLIENT_URL,
                     datamart_name=settings.RDM_UPLOADER_CLIENT_DATAMART_NAME,
                     timeout=1,
                     request_retries=1,
+                    file_status=settings.RDM_RESPONSE_FILE_STATUS
                 )
             )
         elif settings.RDM_UPLOADER_CLIENT_USE_PROXY_API:

{edu-rdm-integration-3.5.5 → edu-rdm-integration-3.5.6}/src/edu_rdm_integration/consts.py

@@ -34,3 +34,4 @@ TASK_QUEUE_NAME = 'RDM'
 # Кортеж операций для обновления данных
 UPDATED_OPERATIONS = (EntityLogOperation.CREATE, EntityLogOperation.UPDATE)
 ALL_OPERATIONS = UPDATED_OPERATIONS + (EntityLogOperation.DELETE, )
+

{edu-rdm-integration-3.5.5 → edu-rdm-integration-3.5.6}/src/edu_rdm_integration/export_data/export.py

@@ -64,10 +64,10 @@ from edu_rdm_integration.base import (
 from edu_rdm_integration.consts import (
     REGIONAL_DATA_MART_INTEGRATION_EXPORTING_DATA,
 )
-from edu_rdm_integration.enums import (
-    FileUploadStatusEnum,
+from edu_rdm_integration.export_data.consts import (
+    TOTAL_ATTACHMENTS_SIZE_KEY,
 )
-from edu_rdm_integration.export_data.export_manger import (
+from edu_rdm_integration.export_data.export_manager import (
     ExportQueueSender,
     WorkerSender,
 )
@@ -294,14 +294,6 @@ class BaseExportLatestEntitiesData(BaseExportEntitiesData):
                 modified__gt=F('exporting_sub_stage__ended_at'),
                 modified__lt=last_finished_export_data,
             ),
-            # Связь лога подэтапа выгрузки с логом загрузки данных (ExportingDataSubStageUploaderClientLog)
-            # имеет статус FileUploadStatusEnum.ERROR:
-            model.objects.filter(
-                exporting_sub_stage__ended_at__gt=now - timedelta(days=365),
-                exporting_sub_stage__exportingdatasubstageuploaderclientlog__created__lt=last_finished_export_data,
-                exporting_sub_stage__exportingdatasubstageuploaderclientlog__file_upload_status=(
-                    FileUploadStatusEnum.ERROR),
-            ),
         )
 
         for queryset in querysets_to_update:
@@ -419,16 +411,18 @@ class UploadData(BaseOperationData):
             caches,
         )
         from uploader_client.contrib.rdm.interfaces.configurations import (
+            RegionalDataMartEmulationUploaderConfig,
             RegionalDataMartUploaderConfig,
         )
         if settings.RDM_UPLOADER_CLIENT_ENABLE_REQUEST_EMULATION:
             uploader_client.set_config(
-                RegionalDataMartUploaderConfig(
+                RegionalDataMartEmulationUploaderConfig(
                     interface='uploader_client.contrib.rdm.interfaces.rest.OpenAPIInterfaceEmulation',
                     url=settings.RDM_UPLOADER_CLIENT_URL,
                     datamart_name=settings.RDM_UPLOADER_CLIENT_DATAMART_NAME,
                     timeout=1,
                     request_retries=1,
+                    file_status=settings.RDM_RESPONSE_FILE_STATUS
                 )
             )
         elif settings.RDM_UPLOADER_CLIENT_USE_PROXY_API:
@@ -457,6 +451,19 @@ class UploadData(BaseOperationData):
                 )
             )
 
+    def update_total_queue_size_in_cache(self, received_files_size: int):
+        """Обновление размера файлов в кеш."""
+        with self.data_cache.lock(f'{TOTAL_ATTACHMENTS_SIZE_KEY}:lock', timeout=300):
+            queue_total_file_size = self.data_cache.get(TOTAL_ATTACHMENTS_SIZE_KEY) or 0
+            if queue_total_file_size:
+                queue_total_file_size -= received_files_size
+                if queue_total_file_size > 0:
+                    self.data_cache.set(
+                        TOTAL_ATTACHMENTS_SIZE_KEY,
+                        queue_total_file_size,
+                        timeout=settings.RDM_REDIS_CACHE_TIMEOUT_SECONDS
+                    )
+
     def upload_data(self, *args, **kwargs):
         """
         Запускает отправку данных в витрину.
@@ -476,6 +483,9 @@ class UploadData(BaseOperationData):
                 if sender.entities:
                     self.result['uploaded_entities'] = ','.join(sender.entities)
 
+                if sender.received_file_size:
+                    self.update_total_queue_size_in_cache(sender.received_file_size)
+
         except Exception as err:
             logger.exception(err)
             raise err

edu-rdm-integration-3.5.5/src/edu_rdm_integration/export_data/export_manger.py → edu-rdm-integration-3.5.6/src/edu_rdm_integration/export_data/export_manager.py

@@ -2,6 +2,8 @@ from pathlib import (
     Path,
 )
 from typing import (
+    TYPE_CHECKING,
+    Any,
     Iterable,
     Optional,
 )
@@ -48,11 +50,17 @@ from edu_rdm_integration.export_data.dataclasses import (
 from edu_rdm_integration.export_data.queue import (
     Queue,
 )
+from edu_rdm_integration.helpers import (
+    FAILED_STATUS,
+    UploadStatusHelper,
+)
 from edu_rdm_integration.models import (
+    DataMartRequestStatus,
     ExportingDataSubStage,
     ExportingDataSubStageAttachment,
     ExportingDataSubStageStatus,
     ExportingDataSubStageUploaderClientLog,
+    UploadStatusRequestLog,
 )
 from edu_rdm_integration.redis_cache import (
     AbstractCache,
@@ -184,6 +192,31 @@ class WorkerSender:
     def __init__(self, queue: Queue):
         self.queue = queue
         self.entities = set()
+        self.received_file_size = 0
+
+    def get_file_upload_status(self, request_id: str) -> tuple[Optional[int], Optional[DataMartRequestStatus], 'Entry']:
+        """Возвращает статус файла в витрине по запросу."""
+        file_upload_status = None
+
+        response, log_entry = UploadStatusHelper.send_upload_status_request(request_id)
+        request_status = None
+
+        if response:
+            request_status = DataMartRequestStatus.get_values_to_enum_data().get(response.get('code'))
+
+            if not request_status:
+                logger.error(
+                    'Не удалось определить статус загрузки данных в витрину. Идентификатор загрузки: '
+                    f'{request_id}, данные ответа: {response}',
+                )
+
+        if request_status in FAILED_STATUS:
+            file_upload_status = FileUploadStatusEnum.ERROR
+
+        elif request_status == DataMartRequestStatus.SUCCESSFULLY_PROCESSED:
+            file_upload_status = FileUploadStatusEnum.FINISHED
+
+        return file_upload_status, request_status, log_entry
 
     def send_files(self):
         """Отправка файлов."""
@@ -194,6 +227,7 @@ class WorkerSender:
                 return
 
             result_to_save = []
+            upload_status_requests_to_save = []
             errors = []
             status = ExportingDataSubStageStatus.FINISHED.key
             for file in upload_files:
@@ -241,6 +275,25 @@ class WorkerSender:
                     file_upload_status=file_upload_status,
                     is_emulation=settings.RDM_UPLOADER_CLIENT_ENABLE_REQUEST_EMULATION,
                 )
+
+                if request_id:
+                    updated_file_upload_status, request_status, log_entry = self.get_file_upload_status(request_id)
+                    upload_status_requests_to_save.append(
+                        UploadStatusRequestLog(
+                            upload=sub_stage_uploader_client_log,
+                            entry=log_entry,
+                            request_status_id=getattr(request_status, 'key', None),
+                        )
+                    )
+                    if updated_file_upload_status and updated_file_upload_status != file_upload_status:
+                        sub_stage_uploader_client_log.file_upload_status = updated_file_upload_status
+
+                    if updated_file_upload_status == FileUploadStatusEnum.FINISHED:
+                        self.received_file_size += file.attachment_size
+
+                    if updated_file_upload_status == FileUploadStatusEnum.ERROR:
+                        status = ExportingDataSubStageStatus.PROCESS_ERROR.key
+
                 result_to_save.append(sub_stage_uploader_client_log)
                 if result.error:
                     logger.warning(
@@ -258,6 +311,7 @@ class WorkerSender:
             # Сохраняем информацию об отправке файлов и убираем подэтап с файлами из очереди
             with atomic():
                 ExportingDataSubStageUploaderClientLog.objects.bulk_create(result_to_save)
+                UploadStatusRequestLog.objects.bulk_create(upload_status_requests_to_save)
                 sub_stage.status_id = status
                 sub_stage.save()
 

{edu-rdm-integration-3.5.5 → edu-rdm-integration-3.5.6}/src/edu_rdm_integration/helpers.py

@@ -56,6 +56,7 @@ from edu_rdm_integration.models import (
     DataMartRequestStatus,
     ExportingDataStage,
     ExportingDataStageStatus,
+    ExportingDataSubStageStatus,
     ExportingDataSubStageUploaderClientLog,
     UploadDataCommand,
     UploadStatusRequestLog,
@@ -82,6 +83,13 @@ if TYPE_CHECKING:
     )
 
 
+FAILED_STATUSES = {
+    DataMartRequestStatus.FAILED_PROCESSING,
+    DataMartRequestStatus.REQUEST_ID_NOT_FOUND,
+    DataMartRequestStatus.FLC_ERROR
+}
+
+
 class UploadStatusHelper:
     """Хелпер проверки статуса загрузки данных в витрину."""
 
@@ -160,8 +168,10 @@ class UploadStatusHelper:
                 request_status_id=getattr(request_status, 'key', None),
             )
 
-            if request_status in {DataMartRequestStatus.FAILED_PROCESSING, DataMartRequestStatus.REQUEST_ID_NOT_FOUND}:
+            if request_status in FAILED_STATUSES:
                 upload.file_upload_status = FileUploadStatusEnum.ERROR
+                upload.sub_stage.status_id = ExportingDataSubStageStatus.PROCESS_ERROR.key
+                upload.sub_stage.save()
 
             elif request_status == DataMartRequestStatus.SUCCESSFULLY_PROCESSED:
                 upload.file_upload_status = FileUploadStatusEnum.FINISHED

edu-rdm-integration-3.5.6/src/edu_rdm_integration/migrations/0015_set_exporting_sub_stage_status.py

@@ -0,0 +1,25 @@
+# Generated by Django 3.2.24 on 2025-02-14 15:49
+
+from django.db import (
+    migrations,
+)
+
+
+def add_exporting_data_sub_stage_status(apps, schema_editor):
+    """Добавление нового статуса этапу выгрузки данных."""
+    ExportingDataSubStageStatus = apps.get_model('edu_rdm_integration', 'ExportingDataSubStageStatus')  # noqa: N806
+    ExportingDataSubStageStatus.objects.get_or_create(
+        key='PROCESS_ERROR',
+        title='Ошибка обработки витриной'
+    )
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('edu_rdm_integration', '0014_uploaddatacommand'),
+    ]
+
+    operations = [
+        migrations.RunPython(add_exporting_data_sub_stage_status, reverse_code=migrations.RunPython.noop),
+    ]

{edu-rdm-integration-3.5.5 → edu-rdm-integration-3.5.6}/src/edu_rdm_integration/models.py

@@ -18,13 +18,13 @@ from django.db.models import (
     DateTimeField,
     FileField,
     ForeignKey,
+    JSONField,
     Manager,
     OneToOneField,
     PositiveIntegerField,
     PositiveSmallIntegerField,
     SmallIntegerField,
     UUIDField,
-    JSONField,
 )
 from django.utils import (
     timezone,
@@ -370,6 +370,9 @@ class ExportingDataSubStageStatus(TitledModelEnum):
     READY_FOR_EXPORT = ModelEnumValue(
         title='Готов к выгрузке',
     )
+    PROCESS_ERROR = ModelEnumValue(
+        title='Ошибка обработки витриной'
+    )
 
     class Meta:
         db_table = 'rdm_exporting_data_sub_stage_status'

{edu-rdm-integration-3.5.5 → edu-rdm-integration-3.5.6}/src/edu_rdm_integration.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: edu-rdm-integration
-Version: 3.5.5
+Version: 3.5.6
 Summary: Интеграция с Региональной витриной данных
 Home-page: 
 Download-URL: 
@@ -31,7 +31,7 @@ Requires-Dist: educommon<4,>=3.11.0
 Requires-Dist: function-tools<1,>=0.9.0
 Requires-Dist: m3-db-utils<1,>=0.3.10
 Requires-Dist: m3-django-compat<2,>=1.10.2
-Requires-Dist: uploader-client<1,>=0.2.1
+Requires-Dist: uploader-client<1,>=0.2.3
 
 # Проект "Интеграция с Региональной витриной данных (РВД)"
 
@@ -300,6 +300,18 @@ Requires-Dist: uploader-client<1,>=0.2.1
 
 ### Удалено
 
+## [3.5.6] - 2025-02-20
+
+Доработана отправка файлов в витрину - добавлена проверка статусов файла и новый статус `Ошибка обработки витриной` для
+ExportingDataSubStageStatus. Доработана задача проверки статусов  UploadStatusHelper.
+
+### Исправлено
+
+- [EDUSCHL-22403](https://jira.bars.group/browse/EDUSCHL-22403)
+  PATCH Добавлен новый статус Ошибка обработки витриной для ExportingDataSubStageStatus. Добавлена проверка 
+  статусов сразу после отправки файлов WorkerSender. Скорректирован статус UploadStatusHelper - при получении статусов 
+  ошибки обработки витриной подэтап помечается как PROCESS_ERROR
+
 
 ## [3.5.5] - 2025-02-11
 

{edu-rdm-integration-3.5.5 → edu-rdm-integration-3.5.6}/src/edu_rdm_integration.egg-info/SOURCES.txt

@@ -115,7 +115,7 @@ src/edu_rdm_integration/export_data/__init__.py
 src/edu_rdm_integration/export_data/consts.py
 src/edu_rdm_integration/export_data/dataclasses.py
 src/edu_rdm_integration/export_data/export.py
-src/edu_rdm_integration/export_data/export_manger.py
+src/edu_rdm_integration/export_data/export_manager.py
 src/edu_rdm_integration/export_data/generators.py
 src/edu_rdm_integration/export_data/helpers.py
 src/edu_rdm_integration/export_data/queue.py
@@ -191,6 +191,7 @@ src/edu_rdm_integration/migrations/0011_exportingdatasubstageentity.py
 src/edu_rdm_integration/migrations/0012_exportingdatasubstageattachment_attachment_size.py
 src/edu_rdm_integration/migrations/0013_set_attachment_size.py
 src/edu_rdm_integration/migrations/0014_uploaddatacommand.py
+src/edu_rdm_integration/migrations/0015_set_exporting_sub_stage_status.py
 src/edu_rdm_integration/migrations/__init__.py
 src/edu_rdm_integration/registry/__init__.py
 src/edu_rdm_integration/registry/actions.py

{edu-rdm-integration-3.5.5 → edu-rdm-integration-3.5.6}/src/edu_rdm_integration.egg-info/requires.txt

@@ -10,4 +10,4 @@ educommon<4,>=3.11.0
 function-tools<1,>=0.9.0
 m3-db-utils<1,>=0.3.10
 m3-django-compat<2,>=1.10.2
-uploader-client<1,>=0.2.1
+uploader-client<1,>=0.2.3