edu-rdm-integration 3.4.0__py3-none-any.whl → 3.4.3__py3-none-any.whl

edu_rdm_integration/app_settings.py

@@ -48,3 +48,9 @@ RDM_UPLOAD_STATUS_TASK_LOCK_EXPIRE_SECONDS = 60 * 60 * 2
 RDM_UPLOAD_QUEUE_MAX_SIZE = 500_000_000
 # Таймаут для сохранения параметров в общем кеш
 RDM_REDIS_CACHE_TIMEOUT_SECONDS = 60 * 60 * 2
+
+# Настройка запуска периодической задачи отправки csv-файлов в витрину:
+RDM_UPLOAD_DATA_TASK_MINUTE = '0'
+RDM_UPLOAD_DATA_TASK_HOUR = '*/2'
+RDM_UPLOAD_DATA_TASK_DAY_OF_WEEK = '*'
+RDM_UPLOAD_DATA_TASK_LOCK_EXPIRE_SECONDS = 60 * 60 * 2

edu_rdm_integration/migrations/0013_set_attachment_size.py

@@ -7,33 +7,42 @@ from django.db import (
 from edu_rdm_integration.enums import (
     FileUploadStatusEnum,
 )
+from educommon.utils.seqtools import (
+    make_chunks,
+)
 
 
 # Значение размера файла по умолчанию (если файл не найден)
 ATTACHMENT_SIZES = 10_485_760
+CHUNK_SIZE = 500
 
 
 def set_attachment_size(apps, schema_editor):
     """Установка размера файла по умолчанию."""
-    ExportingDataSubStageAttachment = apps.get_model('edu_rdm_integration', 'ExportingDataSubStageAttachment')  # noqa: N806
-    ExportingDataSubStageUploaderClientLog = apps.get_model('edu_rdm_integration', 'ExportingDataSubStageUploaderClientLog')  # noqa: N806
+    ExportingDataSubStageAttachment = apps.get_model(
+        'edu_rdm_integration', 'ExportingDataSubStageAttachment'
+    )  # noqa: N806
+    ExportingDataSubStageUploaderClientLog = apps.get_model(
+        'edu_rdm_integration', 'ExportingDataSubStageUploaderClientLog'
+    )  # noqa: N806
 
     attachment_ids = ExportingDataSubStageUploaderClientLog.objects.filter(
         file_upload_status=FileUploadStatusEnum.IN_PROGRESS
-    ).values_list('attachment_id', flat=True)
+    ).values_list('attachment_id', flat=True).iterator()
 
     sub_stage_with_deleted_attachments = []
-    for sub_stage_attachment in ExportingDataSubStageAttachment.objects.filter(
-        attachment_size__isnull=True,
-        id__in=attachment_ids
-    ):
-        if sub_stage_attachment.attachment.field.storage.exists(sub_stage_attachment.attachment.name):
-            sub_stage_attachment.attachment_size = sub_stage_attachment.attachment.size
-        else:
-            sub_stage_attachment.attachment_size = ATTACHMENT_SIZES
-        sub_stage_with_deleted_attachments.append(sub_stage_attachment)
-
-    ExportingDataSubStageAttachment.objects.bulk_update(sub_stage_with_deleted_attachments, ['attachment_size'])
+    for chunked_attachment_ids in make_chunks(attachment_ids, CHUNK_SIZE):
+        for sub_stage_attachment in ExportingDataSubStageAttachment.objects.filter(
+            attachment_size__isnull=True,
+            id__in=chunked_attachment_ids,
+        ):
+            if sub_stage_attachment.attachment.field.storage.exists(sub_stage_attachment.attachment.name):
+                sub_stage_attachment.attachment_size = sub_stage_attachment.attachment.size
+            else:
+                sub_stage_attachment.attachment_size = ATTACHMENT_SIZES
+            sub_stage_with_deleted_attachments.append(sub_stage_attachment)
+
+        ExportingDataSubStageAttachment.objects.bulk_update(sub_stage_with_deleted_attachments, ['attachment_size'])
 
 
 class Migration(migrations.Migration):

edu_rdm_integration/tasks.py

@@ -344,18 +344,18 @@ class TransferLatestEntitiesDataPeriodicTask(UniquePeriodicAsyncTask):
         )
 
 
-class UploadDataAsyncTask(PeriodicAsyncTask):
+class UploadDataAsyncTask(UniquePeriodicAsyncTask):
     """Формирование очереди файлов и их отправка."""
 
     queue = TASK_QUEUE_NAME
     routing_key = TASK_QUEUE_NAME
     description = 'Отправка данных в витрину "Региональная витрина данных"'
-    lock_expire_seconds = settings.RDM_UPLOAD_STATUS_TASK_LOCK_EXPIRE_SECONDS
+    lock_expire_seconds = settings.RDM_UPLOAD_DATA_TASK_LOCK_EXPIRE_SECONDS
     task_type = AsyncTaskType.SYSTEM
     run_every = crontab(
-        minute=settings.RDM_UPLOAD_STATUS_TASK_MINUTE,
-        hour=settings.RDM_UPLOAD_STATUS_TASK_HOUR,
-        day_of_week=settings.RDM_UPLOAD_STATUS_TASK_DAY_OF_WEEK,
+        minute=settings.RDM_UPLOAD_DATA_TASK_MINUTE,
+        hour=settings.RDM_UPLOAD_DATA_TASK_HOUR,
+        day_of_week=settings.RDM_UPLOAD_DATA_TASK_DAY_OF_WEEK,
     )
 
     def process(self, *args, **kwargs):

{edu_rdm_integration-3.4.0.dist-info → edu_rdm_integration-3.4.3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: edu-rdm-integration
-Version: 3.4.0
+Version: 3.4.3
 Summary: Интеграция с Региональной витриной данных
 Home-page: 
 Download-URL: 
@@ -302,6 +302,31 @@ Requires-Dist: uploader-client <1,>=0.2.1
 ### Удалено
 
 
+## 3.4.3 - 2024-10-17
+Исправлена миграция простановки размеров файлов выгрузки в модель ExportingDataSubStageAttachment
+
+
+## Исправлено
+- [EDUSCHL-22651](https://jira.bars.group/browse/EDUSCHL-22651)
+  Исправлена миграция простановки размеров файлов выгрузки в модель ExportingDataSubStageAttachment
+
+
+## 3.4.2 - 2024-10-15
+Добавлены отдельные параметры по управлению временем запуска UploadDataAsyncTask
+
+## Добавлено
+- [EDUSCHL-22166](https://jira.bars.group/browse/EDUSCHL-22166)
+  Добавлены отдельные параметры по управлению временем запуска UploadDataAsyncTask
+
+
+## 3.4.1 - 2024-09-26
+Добавлена документация по сбору/выгрузке сущностей для ЭДС, ЭК
+
+## Добавлено
+- [EDUKNDG-14518](https://jira.bars.group/browse/EDUKNDG-14518)
+  Добавлена документация по сбору/выгрузке сущностей для ЭДС, ЭК
+
+
 ## 3.4.0 - 2024-09-25
 В базовый класс тестирования функций сбора добавлена поддержка формата PhoneNumber для формирования
 экземпляра модели AuditLog.
@@ -317,6 +342,12 @@ Requires-Dist: uploader-client <1,>=0.2.1
    PATCH Вынесены утилиты и базовый класс для логирования и сбора по модели в рамках сущности
 
 
+## 3.3.8 - 2024-10-14
+Задача UploadDataAsyncTask изменена с PeriodicAsyncTask на UploadDataAsyncTask
+- [EDUSCHL-22519]https://jira.bars.group/browse/EDUSCHL-22519
+  PATCH Задача UploadDataAsyncTask изменена с PeriodicAsyncTask на UploadDataAsyncTask
+
+
 ## 3.3.7 - 2024-09-06
 Добавлены команда (UploadEntitiesData) и модель (UploadDataCommand) для логирования и запуска выгрузки данных в витрину
 - [EDUSCHL-22042](https://jira.bars.group/browse/EDUSCHL-22042)

{edu_rdm_integration-3.4.0.dist-info → edu_rdm_integration-3.4.3.dist-info}/RECORD

@@ -1,6 +1,6 @@
 edu_rdm_integration/__init__.py,sha256=fVCvQ7QGI_iCyAeE8dMapyY8gOM617ye5GQqAVGPlZI,72
 edu_rdm_integration/app_meta.py,sha256=v5IU69yaeLbyHF0Ln6iPN_IfizbtF3rCWrz2n71m8dU,337
-edu_rdm_integration/app_settings.py,sha256=bR8zGz2fJPSia5LO4KDUENIlxdqHmKkTAg92-q0qhew,2398
+edu_rdm_integration/app_settings.py,sha256=iqoGMJXFvINd4YwDfXCKjZEsGJCCNxpIRWfvwLrvpPA,2689
 edu_rdm_integration/apps.py,sha256=BoXiTr8B04ZtDxHNXc1Jy2xzRoaq8CMBQ9cfmBK73j4,3571
 edu_rdm_integration/base.py,sha256=_G0qPTAXe6bXfgDHNiZMSsYt3sMuUhLKnHuQCWSFttU,1341
 edu_rdm_integration/consts.py,sha256=Qt52SOCQ-3wOet-_6inJih_W9nToORKXbkxb3jVSjEo,1079
@@ -12,7 +12,7 @@ edu_rdm_integration/models.py,sha256=YP_vot6PbQ3jWv3NNW1wa0gScjOsT2lgdrj7z5V6NHw
 edu_rdm_integration/redis_cache.py,sha256=GZhtM1d0cVr5TEqxh15K7dS371Msit6wRemIiYb2rzk,1548
 edu_rdm_integration/signals.py,sha256=3eRlpkDcFCF6TN80-QM8yBYLcyozzcmoPjz6r4_ApWg,73
 edu_rdm_integration/storages.py,sha256=o5WqUG7SnkeuMt-z8spUi-IraivST-7KHzfY-M3v7FA,6807
-edu_rdm_integration/tasks.py,sha256=QAOytSCEoWtFWJqmTkj3N6NRl_Hds9EXyRfngFHrbPk,16046
+edu_rdm_integration/tasks.py,sha256=9LzUCuCmSR2dDi35SI2VeOdxTpKf33SkFCffov3WO7Y,16044
 edu_rdm_integration/typing.py,sha256=6GOZfRDqOjN-o5NR86-f2o4uXQm_8AUCRRdQTITtrcs,785
 edu_rdm_integration/utils.py,sha256=QMiohtOaId3Zab91IZPazGzhce5pa5e-ilRgXpISkHk,12249
 edu_rdm_integration/adapters/__init__.py,sha256=cU0swn4Ny5ZQz5buWRcWsT1mpWuUFJaUlHf2l7TtEBo,83
@@ -162,7 +162,7 @@ edu_rdm_integration/migrations/0009_auto_20240522_1619.py,sha256=hz8cBq6m18XK5Pb
 edu_rdm_integration/migrations/0010_transferredentity_export_enabled.py,sha256=LJH7QY_1GLWSHLRPB--9PPypPiQd4pAT2-oIOEF84DY,507
 edu_rdm_integration/migrations/0011_exportingdatasubstageentity.py,sha256=2BfIif_hkFv1h6VEfe0Ys4J_uk6LR9YtO711ocDYPso,1263
 edu_rdm_integration/migrations/0012_exportingdatasubstageattachment_attachment_size.py,sha256=y_JQO69k9pEfrJyimaRiAOBmhaJmssIyepCGd-Sy9hs,511
-edu_rdm_integration/migrations/0013_set_attachment_size.py,sha256=Pj_n-ytsC0lhyU67qvH8UHHQ-c-TH5MZFfy-UF6y6M4,1809
+edu_rdm_integration/migrations/0013_set_attachment_size.py,sha256=8-AJpnIViGUh9Q2aMegUPqyw-fthcL-7Ppbm7IvlRms,2047
 edu_rdm_integration/migrations/0014_uploaddatacommand.py,sha256=Hh0vKKiGgKOvY1kBAcmway4dSYUXwVArHAc9YrsjCIU,2079
 edu_rdm_integration/migrations/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 edu_rdm_integration/registry/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -176,9 +176,9 @@ edu_rdm_integration/uploader_log/enums.py,sha256=rgSO3BL2rh2xpfm0Pt4waQW8fB1VMJL
 edu_rdm_integration/uploader_log/managers.py,sha256=y5wTSMzF9hpOpIU_A7nIafL_LBU3QEie6LAYWoB-pBQ,3203
 edu_rdm_integration/uploader_log/ui.py,sha256=YM9Buqp2wxE95Wf5gvAATBzuYzDOossK1sEmvFk07cI,2110
 edu_rdm_integration/uploader_log/templates/ui-js/object-grid-buttons.js,sha256=2xyGe0wdVokM0RhpzRzcRvJPBkBmPe3SlZry4oP4Nzs,6201
-edu_rdm_integration-3.4.0.dist-info/LICENSE,sha256=uw43Gjjj-1vXWCItfSrNDpbejnOwZMrNerUh8oWbq8Q,3458
-edu_rdm_integration-3.4.0.dist-info/METADATA,sha256=yyTvi9MApMnWTUi9cv5G3MRQ8CKcEb48KFVSvcGQXzk,73632
-edu_rdm_integration-3.4.0.dist-info/WHEEL,sha256=yQN5g4mg4AybRjkgi-9yy4iQEFibGQmlz78Pik5Or-A,92
-edu_rdm_integration-3.4.0.dist-info/namespace_packages.txt,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
-edu_rdm_integration-3.4.0.dist-info/top_level.txt,sha256=nRJV0O14UtNE-jGIYG03sohgFnZClvf57H5m6VBXe9Y,20
-edu_rdm_integration-3.4.0.dist-info/RECORD,,
+edu_rdm_integration-3.4.3.dist-info/LICENSE,sha256=uw43Gjjj-1vXWCItfSrNDpbejnOwZMrNerUh8oWbq8Q,3458
+edu_rdm_integration-3.4.3.dist-info/METADATA,sha256=KrZldin3ISdx9GGgOgSj6D9nsJ8HkBkEAMrKTAQJPxc,75074
+edu_rdm_integration-3.4.3.dist-info/WHEEL,sha256=yQN5g4mg4AybRjkgi-9yy4iQEFibGQmlz78Pik5Or-A,92
+edu_rdm_integration-3.4.3.dist-info/namespace_packages.txt,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
+edu_rdm_integration-3.4.3.dist-info/top_level.txt,sha256=nRJV0O14UtNE-jGIYG03sohgFnZClvf57H5m6VBXe9Y,20
+edu_rdm_integration-3.4.3.dist-info/RECORD,,