educommon 3.19.1__py3-none-any.whl → 3.21.0__py3-none-any.whl

educommon/audit_log/management/commands/removing_false_logs.py

@@ -0,0 +1,162 @@
+"""
+Команда удаления ложных логов в таблицах audit_log_auditlog и state_log_all.
+"""
+import codecs
+import os
+
+import psycopg2
+from django.apps import (
+    apps,
+)
+from django.core.management.base import (
+    BaseCommand,
+)
+from django.utils.functional import (
+    cached_property,
+)
+
+from psycopg2.extras import (
+    Json,
+)
+
+from educommon.audit_log.constants import (
+    SQL_FILES_DIR,
+)
+from educommon.audit_log.utils import (
+    get_db_connection_params,
+    get_auto_now_fields_by_model,
+)
+
+
+NEW_AUDIT_LOG_REMOVING_FALSE_LOGS_SQL_FILE_NAME = 'new_audit_log_removing_false_logs.sql'
+OLD_AUDIT_LOG_REMOVING_FALSE_LOGS_SQL_FILE_NAME = 'old_audit_log_removing_false_logs.sql'
+NEW_AUDIT_LOG_TABLE_NAME = 'audit_log_auditlog'
+OLD_AUDIT_LOG_TABLE_NAME = 'state_log_all'
+
+
+class Command(BaseCommand):
+    """Команда удаления ложных логов в таблицах audit_log_auditlog и state_log_all."""
+
+    help = 'Команда удаления ложных логов в таблицах audit_log_auditlog и state_log_all..'
+
+    @cached_property
+    def auto_now_fields_new_audit_log(self) -> dict[str, list[str]]:
+        """Возвращает словарь с полями, имеющими auto_now=True, для моделей с флагом need_to_log = True.
+
+        Returns:
+            Словарь, где ключ это название таблицы, а значение это список названий полей имеющими auto_now=True.
+        """
+
+        return get_auto_now_fields_by_model()
+
+    @cached_property
+    def auto_now_fields_old_audit_log(self) -> dict[str, list[str]]:
+        """Возвращает словарь с полями, имеющими auto_now=True, для моделей с флагом need_to_log = True.
+
+        Returns:
+            Словарь, где ключ это название модели, а значение это список названий полей имеющими auto_now=True.
+        """
+
+        return {
+            model.__name__: fields
+            for model in apps.get_models()
+            if (fields := self.auto_now_fields_new_audit_log.get(model._meta.db_table))
+        }
+
+    def _check_exists_table(self, cursor, table_name: str) -> bool:
+        """Запуск SQL-скрипта для проверки существования таблицы.
+
+        Args:
+            cursor: Объект-курсор для работы с базой данных.
+            table_name: Название таблицы БД.
+
+        Returns:
+            Флаг существования таблицы.
+        """
+        exists_table = """
+            SELECT 1 
+            FROM INFORMATION_SCHEMA.TABLES 
+            WHERE TABLE_SCHEMA = 'public' AND TABLE_NAME = '{table_name}';
+        """.format(table_name=table_name)
+
+        cursor.execute(exists_table)
+
+        return bool(cursor.fetchone())
+
+    def _delete_log(self, cursor, table_name: str, sql_file_name: str, auto_now_fields) -> None:
+        """Запуск SQL-скрипта по удалению ложных логов.
+
+        Args:
+            cursor: Объект-курсор для работы с базой данных.
+            table_name: Название таблицы БД.
+            sql_file_name: Название SQL файла.
+            auto_now_fields: Словарь с полями, имеющими auto_now=True, для моделей с флагом need_to_log = True.
+        """
+        if self._check_exists_table(cursor, table_name):
+            self.stdout.write(f'Удаление ложных записей в таблице {table_name}.')
+            cursor.execute(self._prepare_sql(sql_file_name, auto_now_fields, table_name))
+            self.stdout.write(self.style.SUCCESS(f'Успешно удалены ложные записи в таблице {table_name}.'))
+        else:
+            self.stdout.write(self.style.ERROR(f'Таблица {table_name} не найдена.'))
+
+    def _read_sql(self, sql_file_name: str) -> str:
+        """Чтение SQL-кода из файла.
+
+        Args:
+            sql_file_name: Название SQL файла.
+
+        Returns:
+            SQL-кода из файла.
+        """
+        sql_file_path = os.path.join(SQL_FILES_DIR, sql_file_name)
+
+        with codecs.open(sql_file_path, 'r', 'utf-8') as sql_file:
+            sql = sql_file.read()
+
+        self.stdout.write(f'Выполнение {sql_file_name}.')
+
+        return sql
+
+    def _prepare_sql(self, sql_file_name: str, auto_now_fields: dict[str, list[str]], table_name: str) -> str:
+        """Подготовка SQL-кода.
+
+        Args:
+            sql_file_name: Название SQL файла.
+            auto_now_fields: Словарь с полями, имеющими auto_now=True, для моделей с флагом need_to_log = True.
+            table_name: Название таблицы БД.
+
+        Returns:
+            SQL-кода из файла, с добавленными параметрами.
+        """
+        self.stdout.write(f'Подготовка SQL-кода для удаления ложных логов в таблице {table_name}.')
+
+        params = {
+            'auto_now_fields': Json(auto_now_fields),
+        }
+
+        return self._read_sql(sql_file_name).format(**params)
+
+    def handle(self, *args, **options) -> None:
+        """Формирование SQL-кода и его исполнение."""
+        self.stdout.write('Начало работы команды.')
+
+        connection = psycopg2.connect(**get_db_connection_params())
+        cursor = connection.cursor()
+
+        # Удаление ложных записей в таблице audit_log_auditlog.
+        self._delete_log(
+            cursor=cursor,
+            table_name=NEW_AUDIT_LOG_TABLE_NAME,
+            sql_file_name=NEW_AUDIT_LOG_REMOVING_FALSE_LOGS_SQL_FILE_NAME,
+            auto_now_fields=self.auto_now_fields_new_audit_log,
+        )
+        # Удаление ложных записей в таблице state_log_all.
+        self._delete_log(
+            cursor=cursor,
+            table_name=OLD_AUDIT_LOG_TABLE_NAME,
+            sql_file_name=OLD_AUDIT_LOG_REMOVING_FALSE_LOGS_SQL_FILE_NAME,
+            auto_now_fields=self.auto_now_fields_old_audit_log,
+        )
+
+        connection.commit()
+        self.stdout.write(self.style.SUCCESS('Выполнение команды завершено.'))

educommon/audit_log/sql/new_audit_log_removing_false_logs.sql

@@ -0,0 +1,25 @@
+DO $$
+DECLARE
+    chunk_size INT := 1000;
+    deleted_count INT := 0;
+BEGIN
+    LOOP
+        DELETE FROM audit_log_auditlog
+        WHERE id IN (
+            SELECT a.id
+            FROM audit_log_auditlog a
+            LEFT JOIN audit_log_table t ON a.table_id = t.id
+            WHERE EXISTS (
+                SELECT 1
+                FROM jsonb_each({auto_now_fields}) AS mapping(table_name, keys)
+                WHERE
+                    t.name = mapping.table_name::text AND
+                    akeys(a.changes) <@ ARRAY(SELECT jsonb_array_elements_text(mapping.keys))
+            )
+            LIMIT chunk_size
+        );
+
+        GET DIAGNOSTICS deleted_count = ROW_COUNT;
+        EXIT WHEN deleted_count = 0;
+    END LOOP;
+END $$;

educommon/audit_log/sql/old_audit_log_removing_false_logs.sql

@@ -0,0 +1,78 @@
+-- Функция нахождения разницы между двумя jsonb.
+CREATE OR REPLACE FUNCTION jsonb_diff(val1 JSONB, val2 JSONB)
+RETURNS JSONB AS $$
+DECLARE
+    result JSONB;
+BEGIN
+    SELECT jsonb_object_agg(
+        COALESCE(a.key, b.key),
+        COALESCE(b.value, a.value)
+    )
+    INTO result
+    FROM (
+        SELECT key, value FROM jsonb_each(val1)
+    ) a
+    FULL JOIN (
+        SELECT key, value FROM jsonb_each(val2)
+    ) b ON a.key = b.key
+    WHERE a.value IS DISTINCT FROM b.value
+        OR a.value IS NULL
+        OR b.value IS NULL;
+
+    RETURN result;
+END;
+$$ LANGUAGE plpgsql;
+
+-- Функция проверки, что текстовое поле можно преобразовать в jsonb.
+CREATE OR REPLACE FUNCTION is_valid_jsonb(input_text TEXT)
+RETURNS BOOLEAN AS $$
+BEGIN
+    PERFORM input_text::jsonb;
+    RETURN TRUE;
+EXCEPTION
+    WHEN others THEN
+        RETURN FALSE;
+END;
+$$ LANGUAGE plpgsql IMMUTABLE;
+
+DO $$
+DECLARE
+    chunk_size INT := 1000;
+    deleted_count INT := 0;
+BEGIN
+    LOOP
+        DELETE FROM state_log_all
+        WHERE id IN (
+            SELECT object_previous_and_current.id
+            FROM (
+                SELECT *,
+                    LAG(object) OVER (
+                        PARTITION BY model_id, model
+                        ORDER BY date
+                    ) AS previous_object
+                    FROM state_log_all
+            ) AS object_previous_and_current
+            WHERE previous_object IS NOT NULL
+            AND EXISTS (
+                SELECT 1
+                FROM jsonb_each({auto_now_fields}) AS mapping(table_name, keys)
+                WHERE
+                    is_valid_jsonb(previous_object) AND
+                    is_valid_jsonb(object) AND
+                    model = mapping.table_name::text AND
+                    ARRAY(
+                        SELECT jsonb_object_keys(
+                            jsonb_diff(
+                                COALESCE(previous_object::jsonb -> 0 -> 'fields', jsonb_build_object()),
+                                COALESCE(object::jsonb -> 0 -> 'fields', jsonb_build_object())
+                            )
+                        )
+                    ) <@ ARRAY(SELECT jsonb_array_elements_text(mapping.keys))
+            )
+            LIMIT chunk_size
+        );
+
+        GET DIAGNOSTICS deleted_count = ROW_COUNT;
+        EXIT WHEN deleted_count = 0;
+    END LOOP;
+END $$;

educommon/integration_entities/entities.py

@@ -20,6 +20,8 @@ class BaseEntity:
 
     # Проводимая операция с записью сущности
     operation: int
+    # Id выгружаемой модели
+    id: str
 
     @classmethod
     @abstractmethod

{educommon-3.19.1.dist-info → educommon-3.21.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: educommon
-Version: 3.19.1
+Version: 3.21.0
 Summary: Общая кодовая база для проектов БЦ Образование
 Author-email: BARS Group <education_dev@bars-open.ru>
 Project-URL: Homepage, https://stash.bars-open.ru/projects/EDUBASE/repos/educommon/browse

{educommon-3.19.1.dist-info → educommon-3.21.0.dist-info}/RECORD

@@ -59,6 +59,7 @@ educommon/audit_log/management/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5
 educommon/audit_log/management/commands/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 educommon/audit_log/management/commands/audit_log_migrate_data.py,sha256=J_ZIpamGVBcotc-6cYsz37AJADEEyuiUs-VVyKON47o,12695
 educommon/audit_log/management/commands/reinstall_audit_log.py,sha256=xr56lxpMtxR-KB3lKOmqYVmJ0aUOqzYSeod_6XYQmjc,3722
+educommon/audit_log/management/commands/removing_false_logs.py,sha256=xPZom-wUopJNsDKdCrAWvbFea8wZ2BE71CnVEVMHzgg,6578
 educommon/audit_log/migrations/0001_initial.py,sha256=HDhvBNyVSx_NlFmyA-t_ooFo_TiKf0UHNCZp1GOpLA8,6115
 educommon/audit_log/migrations/0002_install_audit_log.py,sha256=kAhtd1Xz8b6g33wmxBQyRBJIl-LGJdOc7yFy5yhoRJ8,2825
 educommon/audit_log/migrations/0003_logproxy.py,sha256=fx6nCoDtr3bZTtlwiOKP4QTVPxnnqyYDPMAT20Bv4n8,391
@@ -72,6 +73,8 @@ educommon/audit_log/migrations/0010_alter_auditlog_time.py,sha256=XmujmdgNADMy4O
 educommon/audit_log/migrations/__init__.py,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
 educommon/audit_log/sql/configure_audit_log.sql,sha256=M3QxNKTZbn-uNRxGDvNxE9iJh1EOQUTIho7rvc3yhlY,1511
 educommon/audit_log/sql/install_audit_log.sql,sha256=EhyhGMYn8aYCGEBNC2AC7kuG5vfJiAPxhwTMq_2NADE,14505
+educommon/audit_log/sql/new_audit_log_removing_false_logs.sql,sha256=s58p9L1TF9GIC957X7PSfhz60YIt60aNxcIZcoEB1Kc,739
+educommon/audit_log/sql/old_audit_log_removing_false_logs.sql,sha256=7hA5y7-xc1N7cJHpENl07NMmZ48XLPOquI_gqPvupGg,2425
 educommon/audit_log/utils/__init__.py,sha256=FKktT7EeO3M1wqYYSnsgEQ6pSyY4OEzSAXP87ZitI5Q,17239
 educommon/audit_log/utils/operations.py,sha256=skxL7wE4Jx1XlNdPx-Pl3SfiZ1G9jBmcZrXKSQDUGzw,2555
 educommon/auth/__init__.py,sha256=xkGJgqQ5QaEU86n6tJV77ux-2bMTAKbjpVYBCDhcS0E,79
@@ -198,7 +201,7 @@ educommon/importer/validators.py,sha256=8g9PPd25UXDh8ifjna-xVAca4iv63Lq-0SDC7KtA
 educommon/integration_entities/README.rst,sha256=Vc5BYEHY8KfRZVHJg4GAAk0aKvJn52PJuShSxvDMa8g,158
 educommon/integration_entities/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 educommon/integration_entities/consts.py,sha256=XMSR-9Q5cLSXe6Bn0JWFQQ8UfTyJfs7LYT1klj0Xvsc,448
-educommon/integration_entities/entities.py,sha256=v4QQPb8NeQJMdgxjfEcY9_NmPGasQbRPoA10gZEMAjU,2005
+educommon/integration_entities/entities.py,sha256=Qbi3Zl4cYsX81brVLTZPl4mSxW7tDcE_frbj1Xyw0zs,2062
 educommon/integration_entities/enums.py,sha256=MPS3F402TMUP9wDJXBgNHoktVOf-zM8lAD22QphnLtU,335
 educommon/integration_entities/helpers.py,sha256=KEdLGfuBWHfMoPKP93sy2FewK6tsWvS9jFG6Gj8YtlQ,9892
 educommon/integration_entities/mixins.py,sha256=pTZbuLMMsD4y513j-UoRKAjmPjCiwMp8d8I5xWoU33M,1115
@@ -352,7 +355,7 @@ educommon/ws_log/smev/exceptions.py,sha256=VNfzNHlj5Pz8D4979d_msTkxC-RQVoMctsgoJ
 educommon/ws_log/templates/report/smev_logs.xlsx,sha256=nnYgB0Z_ix8HoxsRICjsZfFRQBdra-5Gd8nWhCxTjYg,10439
 educommon/ws_log/templates/ui-js/smev-logs-list-window.js,sha256=AGup3D8GTJSY9WdDPj0zBJeYQBFOmGgcbxPOJbKK-nY,513
 educommon/ws_log/templates/ui-js/smev-logs-report-setting-window.js,sha256=nQ7QYK9frJcE7g7kIt6INg9TlEEJAPPayBJgRaoTePA,1103
-educommon-3.19.1.dist-info/METADATA,sha256=bTmxkUclBTbBh8nVHUJdEpjKH3hzPyKp6WeSHzmOy_U,2380
-educommon-3.19.1.dist-info/WHEEL,sha256=1tXe9gY0PYatrMPMDd6jXqjfpz_B-Wqm32CPfRC58XU,91
-educommon-3.19.1.dist-info/top_level.txt,sha256=z5fbW7bz_0V1foUm_FGcZ9_MTpW3N1dBN7-kEmMowl4,10
-educommon-3.19.1.dist-info/RECORD,,
+educommon-3.21.0.dist-info/METADATA,sha256=ymYMv5xu-FGq3UhMGK7cQPy76COkIvxrvH7_bJH5yZ4,2380
+educommon-3.21.0.dist-info/WHEEL,sha256=1tXe9gY0PYatrMPMDd6jXqjfpz_B-Wqm32CPfRC58XU,91
+educommon-3.21.0.dist-info/top_level.txt,sha256=z5fbW7bz_0V1foUm_FGcZ9_MTpW3N1dBN7-kEmMowl4,10
+educommon-3.21.0.dist-info/RECORD,,