edu-rdm-integration 3.15.2__py3-none-any.whl → 3.15.4__py3-none-any.whl

edu_rdm_integration/core/consts.py

@@ -21,6 +21,14 @@ HASH_ALGORITHM = HashGostFunctionVersion.GOST12_512
 
 BATCH_SIZE = 5000
 
+CHUNK_MAX_VALUE = 100000
+"""Максимальное значение для размера чанка при сборе данных РВД.
+Ограничение 100000 предотвращает чрезмерное потребление памяти и блокировку БД
+при обработке больших объемов данных в одном батче."""
+
+SPLIT_BY_QUANTITY_MAX_VALUE = 366
+"""Максимальное значение для размера подпериода при временном разделении данных."""
+
 ACADEMIC_YEAR = {
     'start_day': 1,
     'start_month': 9,

edu_rdm_integration/stages/collect_data/generators.py

@@ -167,15 +167,15 @@ class BaseFirstCollectModelsDataCommandsGenerator:
                     period_ended_at=self.logs_period_ended_at,
                 )
 
-                for start_datetime, end_datetime in intervals:
-                    params_for_commands.append(
-                        {
-                            'period_started_at': start_datetime,
-                            'period_ended_at': end_datetime,
-                            'model': rdm_model.key,
-                            'generation_id': self.generation_id,
-                        }
-                    )
+                params_for_model = [
+                    {
+                        'period_started_at': start_datetime,
+                        'period_ended_at': end_datetime,
+                        'model': rdm_model.key,
+                        'generation_id': self.generation_id,
+                    }
+                    for start_datetime, end_datetime in intervals
+                ]
 
             # Если подпериод не указан, то формируется список с разбиением по batch_size
             else:
@@ -213,15 +213,32 @@ class BaseFirstCollectModelsDataCommandsGenerator:
                     cursor.execute(temp_get_logs_periods_sql)
                     rows = cursor.fetchall()
 
-                for period_started_at, period_ended_at, batch_number in rows:
-                    params_for_commands.append(
-                        {
-                            'period_started_at': period_started_at,
-                            'period_ended_at': period_ended_at,
-                            'model': rdm_model.key,
-                            'generation_id': self.generation_id,
-                        }
-                    )
+                params_for_model = [
+                    {
+                        'period_started_at': period_started_at,
+                        'period_ended_at': period_ended_at,
+                        'model': rdm_model.key,
+                        'generation_id': self.generation_id,
+                    }
+                    for period_started_at, period_ended_at, _ in rows
+                ]
+
+            if params_for_model:
+                # Корректируем границы начала и конца сбора данных под значения введенные пользователем
+                params_for_model[0]['period_started_at'] = self.logs_period_started_at
+                params_for_model[-1]['period_ended_at'] = self.logs_period_ended_at
+            else:
+                # Создаем команду для отображения в UI, даже если не нашлось данных для обработки
+                params_for_model.append(
+                    {
+                        'period_started_at': self.logs_period_started_at,
+                        'period_ended_at': self.logs_period_ended_at,
+                        'model': rdm_model.key,
+                        'generation_id': self.generation_id,
+                    }
+                )
+
+            params_for_commands.extend(params_for_model)
 
         return params_for_commands
 

edu_rdm_integration/stages/collect_data/registry/actions.py

@@ -2,6 +2,9 @@ from functools import (
     partial,
 )
 
+from django.conf import (
+    settings,
+)
 from django.db.models import (
     F,
     Func,
@@ -268,6 +271,4 @@ class BaseCollectingDataProgressPack(BaseCommandProgressPack):
             )
             for command in commands_to_save
         ]
-
-        for obj in objs:
-            super().save_row(obj, create_new, request, context, *args, **kwargs)
+        self.model.objects.bulk_create(objs, batch_size=settings.COLLECT_PROGRESS_BATCH_SIZE)

edu_rdm_integration/stages/collect_data/registry/templates/ui-js/collect-command-window.js

@@ -0,0 +1,77 @@
+{% load educommon %}
+{% include "ui-js/validators.js" %}
+
+Ext.onReady(function() {
+    // Инициализация валидаторов
+    initializeValidators();
+
+    // Инициализация взаимозависимых полей
+    initializeBatchSizeSplitByLogic();
+});
+
+function initializeValidators() {
+    // Валидатор для institute_ids
+    var instituteIdsField = Ext.getCmp('institute_ids');
+    if (instituteIdsField) {
+        instituteIdsField.validate = instituteIdsValidator;
+        instituteIdsField.on('blur', function() {
+            this.validate();
+        });
+    }
+
+    // Валидатор для institute_count
+    var instituteCountField = Ext.getCmp('institute_count');
+    if (instituteCountField) {
+        var originalValidator = instituteCountField.validate;
+
+        instituteCountField.validate = function() {
+            if (!instituteCountValidator.call(this)) {
+                return false;
+            }
+            if (originalValidator) {
+                return originalValidator.call(this);
+            }
+            return true;
+        };
+
+        instituteCountField.on('blur', function() {
+            this.validate();
+        });
+    }
+}
+
+function initializeBatchSizeSplitByLogic() {
+    var batchSizeField = Ext.getCmp('batch_size');
+    var splitByField = Ext.getCmp('split_by');
+
+    if (!batchSizeField || !splitByField) {
+        return;
+    }
+
+    // Функция для обновления обязательности полей
+    function updateFieldRequirements() {
+        var batchSizeValue = batchSizeField.getValue();
+        var splitByValue = splitByField.getValue();
+
+        if (splitByValue && splitByValue !== '') {
+            batchSizeField.allowBlank = true;
+            batchSizeField.clearInvalid();
+        } else {
+            batchSizeField.allowBlank = false;
+        }
+
+        if (batchSizeValue && batchSizeValue !== '' && batchSizeValue !== 0) {
+            splitByField.allowBlank = true;
+            splitByField.clearInvalid();
+        } else {
+            splitByField.allowBlank = false;
+        }
+    }
+
+    // Обработчики изменения полей
+    batchSizeField.on('change', updateFieldRequirements);
+    splitByField.on('change', updateFieldRequirements);
+
+    // Инициализация при загрузке
+    updateFieldRequirements();
+}

edu_rdm_integration/stages/collect_data/registry/templates/ui-js/create-collect-command-win.js

@@ -0,0 +1,14 @@
+var win = Ext.getCmp('{{ component.client_id }}');
+var form = win.getForm();
+var logsPeriodEndField = form.findField('logs_period_ended_at');
+
+// Устанавливаем текущую дату и время как максимальное значение
+// и выполняем валидацию каждую секунду
+var validationInterval = setInterval(function() {
+    logsPeriodEndField.setMaxValue(new Date())
+    logsPeriodEndField.validate();
+}, 1000);
+
+win.on('destroy', function() {
+    clearInterval(validationInterval);
+});

edu_rdm_integration/stages/collect_data/registry/templates/ui-js/validators.js

@@ -0,0 +1,51 @@
+// Валидатор для поля institute_ids
+var instituteIdsValidator = function () {
+    var value = this.getValue();
+    
+    // Если поле пустое - это допустимо
+    if (!value || value.trim() === '') {
+        this.clearInvalid();
+        return true;
+    }
+
+    value = value.trim();
+
+    if (!/^[0-9,]+$/.test(value)) {
+        this.markInvalid('Разрешены только цифры и запятые');
+        return false;
+    }
+
+    var numbers = value.split(',');
+
+    for (var i = 0; i < numbers.length; i++) {
+        if (numbers[i].trim() === '') {
+            this.markInvalid('Между запятыми не должно быть пустых значений');
+            return false;
+        }
+    }
+
+    var uniqueNumbers = [];
+    for (var j = 0; j < numbers.length; j++) {
+        var num = parseInt(numbers[j].trim());
+        if (uniqueNumbers.indexOf(num) !== -1) {
+            this.markInvalid('ID организаций не должны повторяться');
+            return false;
+        }
+        uniqueNumbers.push(num);
+    }
+    
+    this.clearInvalid();
+    return true;
+};
+
+// Валидатор для поля institute_count - запрещаем 0
+var instituteCountValidator = function() {
+    var value = this.getValue();
+    if (value === 0) {
+        this.markInvalid('Количество организаций не может быть равно 0');
+        return false;
+    }
+    
+    this.clearInvalid();
+    return true;
+};

edu_rdm_integration/stages/collect_data/registry/ui.py

@@ -24,6 +24,8 @@ from educommon.utils.date import (
 
 from edu_rdm_integration.core.consts import (
     BATCH_SIZE,
+    CHUNK_MAX_VALUE,
+    SPLIT_BY_QUANTITY_MAX_VALUE,
 )
 from edu_rdm_integration.core.registry.ui import (
     BaseCreateCommandWindow,
@@ -39,6 +41,11 @@ from edu_rdm_integration.stages.collect_data.consts import (
 class CreateCollectCommandWindow(BaseCreateCommandWindow):
     """Окно создания команды сбора данных модели РВД."""
 
+    def __init__(self):
+        super().__init__()
+
+        self.template_globals = 'ui-js/create-collect-command-win.js'
+
     def _init_components(self):
         """Инициализация компонентов."""
         super()._init_components()
@@ -70,8 +77,9 @@ class CreateCollectCommandWindow(BaseCreateCommandWindow):
             display_field='split_by',
             label='Единица подпериода',
             anchor='100%',
-            editable=True,
+            editable=False,
             trigger_action_all=True,
+            client_id='split_by',
         )
         split_by.set_store(ExtDataStore(enumerate(DatesSplitter.get_split_by_modes())))
         split_by_quantity = ExtNumberField(
@@ -80,6 +88,7 @@ class CreateCollectCommandWindow(BaseCreateCommandWindow):
             allow_blank=False,
             allow_decimals=False,
             allow_negative=False,
+            max_value=SPLIT_BY_QUANTITY_MAX_VALUE,
             anchor='100%',
             value=1,
         )
@@ -104,6 +113,8 @@ class CreateCollectCommandWindow(BaseCreateCommandWindow):
             allow_negative=False,
             anchor='100%',
             value=BATCH_SIZE,
+            max_value=CHUNK_MAX_VALUE,
+            client_id='batch_size',
         )
         by_institutes = ExtCheckBox(
             anchor='100%',
@@ -111,10 +122,12 @@ class CreateCollectCommandWindow(BaseCreateCommandWindow):
             name='by_institutes',
         )
         institute_ids = ExtStringField(
-            label='id организаций',
+            label='ID организаций (через запятую, например: 1,2,3 или оставить пустым для всех)',
             name='institute_ids',
             allow_blank=True,
             anchor='100%',
+            max_length=100,
+            client_id='institute_ids',
         )
         institute_count = ExtNumberField(
             name='institute_count',
@@ -123,6 +136,8 @@ class CreateCollectCommandWindow(BaseCreateCommandWindow):
             anchor='100%',
             min_value=ALL_UNITS_IN_COMMAND,
             value=ALL_UNITS_IN_COMMAND,
+            client_id='institute_count',
+            allow_blank=False,
         )
         hint_text = ExtDisplayField(
             value=(
@@ -151,6 +166,12 @@ class CreateCollectCommandWindow(BaseCreateCommandWindow):
             split_mode,
         )
 
+    def set_params(self, params: Dict[str, Any]) -> None:
+        """Устанавливает параметры окна."""
+        super().set_params(params)
+
+        self.template_globals = 'ui-js/collect-command-window.js'
+
 
 class DetailCollectCommandWindow(BaseEditWindow):
     """Окно просмотра команды сбора данных модели РВД."""

edu_rdm_integration/stages/export_data/registry/templates/ui-js/create-export-command-win.js

@@ -0,0 +1,14 @@
+var win = Ext.getCmp('{{ component.client_id }}');
+var form = win.getForm();
+var periodEndField = form.findField('period_ended_at');
+
+// Устанавливаем текущую дату и время как максимальное значение
+// и выполняем валидацию каждую секунду
+var validationInterval = setInterval(function() {
+    periodEndField.setMaxValue(new Date())
+    periodEndField.validate();
+}, 1000);
+
+win.on('destroy', function() {
+    clearInterval(validationInterval);
+});

edu_rdm_integration/stages/export_data/registry/ui.py

@@ -44,6 +44,11 @@ from edu_rdm_integration.rdm_entities.models import (
 class CreateExportCommandWindow(BaseCreateCommandWindow):
     """Окно создания команды экспорта данных сущности РВД."""
 
+    def __init__(self):
+        super().__init__()
+
+        self.template_globals = 'ui-js/create-export-command-win.js'
+
     def _init_components(self):
         """Инициализация компонентов."""
         super()._init_components()

{edu_rdm_integration-3.15.2.dist-info → edu_rdm_integration-3.15.4.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: edu-rdm-integration
-Version: 3.15.2
+Version: 3.15.4
 Summary: Интеграция с Региональной витриной данных
 Author-email: BARS Group <education_dev@bars.group>
 Project-URL: Homepage, https://stash.bars-open.ru/projects/EDUBASE/repos/edu-rdm-integration/browse

{edu_rdm_integration-3.15.2.dist-info → edu_rdm_integration-3.15.4.dist-info}/RECORD

@@ -9,7 +9,7 @@ edu_rdm_integration/collect_and_export_data/migrations/0002_auto_20250204_1413.p
 edu_rdm_integration/collect_and_export_data/migrations/0003_auto_20250704_0725.py,sha256=IcBTuQWp7D_mP1chCgTXUIJcm5nf2Scdt3ZJgjCgvgI,1108
 edu_rdm_integration/collect_and_export_data/migrations/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 edu_rdm_integration/core/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-edu_rdm_integration/core/consts.py,sha256=5pFDMWNGqKYqgqqUjb_QiUyWnjugHzmpl6beMJgWVEI,955
+edu_rdm_integration/core/consts.py,sha256=4dklnQS7rtuYJu0Hh7oX1Z9dc72pbIWZl9gXMUvc4rI,1526
 edu_rdm_integration/core/enums.py,sha256=HHZxX2vIe-c4SY6BIWgOg-Tu-_LVJjltFZ1lI9YITLw,3824
 edu_rdm_integration/core/helpers.py,sha256=Fe2AyI8kHAfcnXAn2I_SyNraI5uAjNzHObUNtZtum-g,12183
 edu_rdm_integration/core/mapping.py,sha256=1B6TsC4Os9wiM8L8BChnCNv_iWqjeWu3bdDsqKVsId0,616
@@ -77,7 +77,7 @@ edu_rdm_integration/stages/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJ
 edu_rdm_integration/stages/collect_data/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 edu_rdm_integration/stages/collect_data/apps.py,sha256=PhrxzAMXvzXajSlgwLudAgasVjh9IFS5M7a7pGbEKC0,339
 edu_rdm_integration/stages/collect_data/consts.py,sha256=tzaK9oxzdMRq3oTEocPz4umoXSJWUtFc7YhyXucCNbs,127
-edu_rdm_integration/stages/collect_data/generators.py,sha256=k6An9y3_3mzopYkoudmhspHB6sIUSDdLv6JPdMdMwWo,12075
+edu_rdm_integration/stages/collect_data/generators.py,sha256=OM5KsInz_as8K2TX4XLJ-AWTnul-VR4v6oS5ORdU0fI,12970
 edu_rdm_integration/stages/collect_data/helpers.py,sha256=xy8z9yJKEMjNUPNhrsRRtnYy6RVbwDoD5zSDAX7y_6U,5260
 edu_rdm_integration/stages/collect_data/mixins.py,sha256=izioaiPC26BDODgi_Lhy33IaH207945tGjFnbFLMQyI,2072
 edu_rdm_integration/stages/collect_data/models.py,sha256=hxB2JUF0U0d7Gr8UcPDEhONLACTuqA63SFPEK5qn2Mo,8916
@@ -151,9 +151,12 @@ edu_rdm_integration/stages/collect_data/migrations/0003_auto_20250704_0810.py,sh
 edu_rdm_integration/stages/collect_data/migrations/0004_auto_20250704_0825.py,sha256=B6SUsxlhQvWoD8lFGNwaMUCFDzhPj91bsMdmAcSuEDg,1379
 edu_rdm_integration/stages/collect_data/migrations/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 edu_rdm_integration/stages/collect_data/registry/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-edu_rdm_integration/stages/collect_data/registry/actions.py,sha256=5eT6nw4xYuJlfVrFxoqn0SF4PqSgHIbMc7rlx8dWypE,9769
+edu_rdm_integration/stages/collect_data/registry/actions.py,sha256=mUMGv1FjzjY0G46gTgi_mfL0yUjuCJlA2wQutq3jmPo,9798
 edu_rdm_integration/stages/collect_data/registry/apps.py,sha256=K5f97YXKMmdM7m33qgQYvJjrA8_eGAJ4VWyuRjJ0gwQ,439
-edu_rdm_integration/stages/collect_data/registry/ui.py,sha256=zmWZ0qgdu83V2MxRDIPyvDrgRcQKaeEzvMZ2CuvDE1M,7613
+edu_rdm_integration/stages/collect_data/registry/ui.py,sha256=oOghmS49uCOEycOL3Wz-4-NUSy9mp7_pcwXf_VVz8rU,8427
+edu_rdm_integration/stages/collect_data/registry/templates/ui-js/collect-command-window.js,sha256=IWG4CczBG9-Bi6X2Hivod4z63gSBZ6Fd126QVRRFacA,2386
+edu_rdm_integration/stages/collect_data/registry/templates/ui-js/create-collect-command-win.js,sha256=BypYLy6-xKjmQucGy_uVEx2830jh5f7VFvUaAaqL_AY,549
+edu_rdm_integration/stages/collect_data/registry/templates/ui-js/validators.js,sha256=c0p0ND7i2C-fZrADgiUv9Eekjx_ZlFHBrgd-oXuqXKI,1525
 edu_rdm_integration/stages/export_data/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 edu_rdm_integration/stages/export_data/apps.py,sha256=TU6AaMZGQE2oHVwKgtUDzFplaVasb2tMIurhkhwkxZo,406
 edu_rdm_integration/stages/export_data/consts.py,sha256=ZEi1kXMs-54KFKxkyGIQVwZ4d8OrOF_vLFQIjjWdSPQ,441
@@ -206,7 +209,8 @@ edu_rdm_integration/stages/export_data/migrations/__init__.py,sha256=47DEQpj8HBS
 edu_rdm_integration/stages/export_data/registry/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 edu_rdm_integration/stages/export_data/registry/actions.py,sha256=r-NAPs4pYVectt_J38UzMsnKrWfYkjy2rhx3LUGKQWk,10423
 edu_rdm_integration/stages/export_data/registry/apps.py,sha256=71DtJQ2ULt8_3CnTu2VAfT5ABBrDNY1nKTmZ6UtvIpw,448
-edu_rdm_integration/stages/export_data/registry/ui.py,sha256=W5hFTBBgIPc7q0PAK8ObENxOYb5iq1SnV0-DZbiL4-U,5920
+edu_rdm_integration/stages/export_data/registry/ui.py,sha256=GPp52ziV_xb0cnveprqIYmrx4kv0zgvrhMg-PHqUFeE,6042
+edu_rdm_integration/stages/export_data/registry/templates/ui-js/create-export-command-win.js,sha256=ADE04kltbx1e0spaFK_ycBmtxBQPz6pqfaNY_czo9z8,532
 edu_rdm_integration/stages/export_data/registry/templates/ui-js/stage_for_export.js,sha256=329OZIpiKHlQ-i8JStjBLXtouIMKuJHbycArUGSskfk,737
 edu_rdm_integration/stages/service/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 edu_rdm_integration/stages/service/apps.py,sha256=lgCG4_kpwgfDWh6y-GNuUwz5SOjkP7oS8kkUyVUcNRg,648
@@ -242,8 +246,8 @@ edu_rdm_integration/stages/upload_data/uploader_log/ui.py,sha256=mU3XA9zVKHGqzNk
 edu_rdm_integration/stages/upload_data/uploader_log/migrations/0001_initial.py,sha256=r5oOB7DBK9-mfuqPAgjXUJY5-hEcmMdILCwDTpaLnBc,753
 edu_rdm_integration/stages/upload_data/uploader_log/migrations/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 edu_rdm_integration/stages/upload_data/uploader_log/templates/ui-js/object-grid-buttons.js,sha256=2xyGe0wdVokM0RhpzRzcRvJPBkBmPe3SlZry4oP4Nzs,6201
-edu_rdm_integration-3.15.2.dist-info/licenses/LICENSE,sha256=uw43Gjjj-1vXWCItfSrNDpbejnOwZMrNerUh8oWbq8Q,3458
-edu_rdm_integration-3.15.2.dist-info/METADATA,sha256=IPzmLm3bBn0vH34ibUeQuYNwOHfq1vYACpPlE4mFKDg,39741
-edu_rdm_integration-3.15.2.dist-info/WHEEL,sha256=1tXe9gY0PYatrMPMDd6jXqjfpz_B-Wqm32CPfRC58XU,91
-edu_rdm_integration-3.15.2.dist-info/top_level.txt,sha256=nRJV0O14UtNE-jGIYG03sohgFnZClvf57H5m6VBXe9Y,20
-edu_rdm_integration-3.15.2.dist-info/RECORD,,
+edu_rdm_integration-3.15.4.dist-info/licenses/LICENSE,sha256=uw43Gjjj-1vXWCItfSrNDpbejnOwZMrNerUh8oWbq8Q,3458
+edu_rdm_integration-3.15.4.dist-info/METADATA,sha256=hkEr7M93oXb0udY1lgc4_zF_YdGnuH9wEZStdA4xV00,39741
+edu_rdm_integration-3.15.4.dist-info/WHEEL,sha256=1tXe9gY0PYatrMPMDd6jXqjfpz_B-Wqm32CPfRC58XU,91
+edu_rdm_integration-3.15.4.dist-info/top_level.txt,sha256=nRJV0O14UtNE-jGIYG03sohgFnZClvf57H5m6VBXe9Y,20
+edu_rdm_integration-3.15.4.dist-info/RECORD,,