edu-rdm-integration 3.22.1__py3-none-any.whl → 3.22.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of edu-rdm-integration might be problematic. Click here for more details.
- edu_rdm_integration/rdm_entities/models.py +5 -0
- edu_rdm_integration/rdm_models/models.py +29 -11
- edu_rdm_integration/stages/collect_data/models.py +5 -0
- edu_rdm_integration/stages/collect_data/registry/actions.py +3 -3
- edu_rdm_integration/stages/export_data/migrations/0003_fix_fk_constraints.py +1 -1
- edu_rdm_integration/stages/export_data/migrations/0004_alter_rdmexportingdatasubstageattachment_exporting_data_sub_stage.py +22 -0
- edu_rdm_integration/stages/export_data/models.py +8 -1
- edu_rdm_integration/stages/export_data/registry/actions.py +3 -3
- edu_rdm_integration/stages/service/service_outdated_data/cleaners/base.py +139 -48
- edu_rdm_integration/stages/service/service_outdated_data/cleaners/export_data.py +22 -1
- {edu_rdm_integration-3.22.1.dist-info → edu_rdm_integration-3.22.3.dist-info}/METADATA +1 -1
- {edu_rdm_integration-3.22.1.dist-info → edu_rdm_integration-3.22.3.dist-info}/RECORD +15 -14
- {edu_rdm_integration-3.22.1.dist-info → edu_rdm_integration-3.22.3.dist-info}/WHEEL +0 -0
- {edu_rdm_integration-3.22.1.dist-info → edu_rdm_integration-3.22.3.dist-info}/licenses/LICENSE +0 -0
- {edu_rdm_integration-3.22.1.dist-info → edu_rdm_integration-3.22.3.dist-info}/top_level.txt +0 -0
|
@@ -70,6 +70,11 @@ class RDMEntityEnum(TitledModelEnum):
|
|
|
70
70
|
|
|
71
71
|
return model_enums
|
|
72
72
|
|
|
73
|
+
@classmethod
|
|
74
|
+
def get_choices(cls) -> list[tuple[str, str]]:
|
|
75
|
+
"""Возвращает список кортежей из ключей и ключей перечисления сущностей."""
|
|
76
|
+
return [(key, key) for key in sorted(cls.get_model_enum_keys())]
|
|
77
|
+
|
|
73
78
|
@classmethod
|
|
74
79
|
def extend(
|
|
75
80
|
cls,
|
|
@@ -35,15 +35,6 @@ from m3_db_utils.models import (
|
|
|
35
35
|
class BaseRDMModel(ReprStrPreModelMixin, BaseObjectModel):
|
|
36
36
|
"""Базовая модель РВД."""
|
|
37
37
|
|
|
38
|
-
collecting_sub_stage = ForeignKey(
|
|
39
|
-
verbose_name='Подэтап сбора данных',
|
|
40
|
-
to='edu_rdm_integration_collect_data_stage.RDMCollectingDataSubStage',
|
|
41
|
-
on_delete=CASCADE,
|
|
42
|
-
)
|
|
43
|
-
operation = SmallIntegerField(
|
|
44
|
-
verbose_name='Действие',
|
|
45
|
-
choices=EntityLogOperation.get_choices(),
|
|
46
|
-
)
|
|
47
38
|
created = DateTimeField(
|
|
48
39
|
verbose_name='Дата создания',
|
|
49
40
|
auto_now_add=True,
|
|
@@ -68,10 +59,32 @@ class BaseRDMModel(ReprStrPreModelMixin, BaseObjectModel):
|
|
|
68
59
|
abstract = True
|
|
69
60
|
|
|
70
61
|
|
|
71
|
-
class
|
|
62
|
+
class BaseAdditionalRDMModel(BaseRDMModel):
|
|
63
|
+
"""Абстрактная вспомогательная модель РВД.
|
|
64
|
+
|
|
65
|
+
Является базовым классом для моделей РВД, которые не являются основными для сущностей РВД. Для таких моделей
|
|
66
|
+
производится сбор данных.
|
|
67
|
+
"""
|
|
68
|
+
|
|
69
|
+
collecting_sub_stage = ForeignKey(
|
|
70
|
+
verbose_name='Подэтап сбора данных',
|
|
71
|
+
to='edu_rdm_integration_collect_data_stage.RDMCollectingDataSubStage',
|
|
72
|
+
on_delete=CASCADE,
|
|
73
|
+
)
|
|
74
|
+
operation = SmallIntegerField(
|
|
75
|
+
verbose_name='Действие',
|
|
76
|
+
choices=EntityLogOperation.get_choices(),
|
|
77
|
+
)
|
|
78
|
+
|
|
79
|
+
class Meta:
|
|
80
|
+
abstract = True
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
class BaseMainRDMModel(BaseAdditionalRDMModel):
|
|
72
84
|
"""Абстрактная основная модель РВД.
|
|
73
85
|
|
|
74
|
-
Является базовым классом для моделей РВД, которые являются основными для сущностей РВД.
|
|
86
|
+
Является базовым классом для моделей РВД, которые являются основными для сущностей РВД. Для таких моделей
|
|
87
|
+
производится сбор и выгрузка данных.
|
|
75
88
|
"""
|
|
76
89
|
|
|
77
90
|
exporting_sub_stage = ForeignKey(
|
|
@@ -103,6 +116,11 @@ class RDMModelEnum(TitledModelEnum):
|
|
|
103
116
|
verbose_name = 'Модель-перечисление моделей "Региональной витрины данных"'
|
|
104
117
|
verbose_name_plural = 'Модели-перечисления моделей "Региональной витрины данных"'
|
|
105
118
|
|
|
119
|
+
@classmethod
|
|
120
|
+
def get_choices(cls) -> list[tuple[str, str]]:
|
|
121
|
+
"""Возвращает список кортежей из ключей и ключей перечисления моделей."""
|
|
122
|
+
return [(key, key) for key in sorted(cls.get_model_enum_keys())]
|
|
123
|
+
|
|
106
124
|
@classmethod
|
|
107
125
|
def _get_model_relations(cls, model: Type['BaseRDMModel']) -> dict[str, str]:
|
|
108
126
|
"""Получение списка связей модели РВД."""
|
|
@@ -65,6 +65,11 @@ class RDMCollectingDataStageStatus(TitledModelEnum):
|
|
|
65
65
|
verbose_name = 'Модель-перечисление статусов этапа сбора данных'
|
|
66
66
|
verbose_name_plural = 'Модели-перечисления статусов этапов сбора данных'
|
|
67
67
|
|
|
68
|
+
@classmethod
|
|
69
|
+
def get_choices(cls) -> list[tuple[str, str]]:
|
|
70
|
+
"""Возвращает список кортежей из ключей и ключей перечисления статусов."""
|
|
71
|
+
return [(key, key) for key in cls.get_model_enum_keys()]
|
|
72
|
+
|
|
68
73
|
|
|
69
74
|
class RDMCollectingDataStage(ReprStrPreModelMixin, BaseObjectModel):
|
|
70
75
|
"""Этап подготовки данных в рамках Функций. За работу Функции отвечает ранер менеджер."""
|
|
@@ -83,7 +83,7 @@ class BaseCollectingDataProgressPack(BaseCommandProgressPack):
|
|
|
83
83
|
'header': 'Модель',
|
|
84
84
|
'sortable': True,
|
|
85
85
|
'filter': ChoicesFilter(
|
|
86
|
-
choices=
|
|
86
|
+
choices=partial(RDMModelEnum.get_choices),
|
|
87
87
|
parser=str,
|
|
88
88
|
lookup=lambda key: Q(model=key) if key else Q(),
|
|
89
89
|
),
|
|
@@ -93,7 +93,7 @@ class BaseCollectingDataProgressPack(BaseCommandProgressPack):
|
|
|
93
93
|
'header': 'Статус асинхронной задачи',
|
|
94
94
|
'sortable': True,
|
|
95
95
|
'filter': ChoicesFilter(
|
|
96
|
-
choices=
|
|
96
|
+
choices=partial(AsyncTaskStatus.get_choices),
|
|
97
97
|
parser=str,
|
|
98
98
|
lookup='task__status_id',
|
|
99
99
|
),
|
|
@@ -118,7 +118,7 @@ class BaseCollectingDataProgressPack(BaseCommandProgressPack):
|
|
|
118
118
|
'header': 'Статус сбора',
|
|
119
119
|
'sortable': True,
|
|
120
120
|
'filter': ChoicesFilter(
|
|
121
|
-
choices=
|
|
121
|
+
choices=partial(RDMCollectingDataStageStatus.get_choices),
|
|
122
122
|
parser=str,
|
|
123
123
|
lookup=lambda key: Q(stage__status=key) if key else Q(),
|
|
124
124
|
),
|
|
@@ -28,7 +28,7 @@ def apply_fk_updates(apps, schema_editor):
|
|
|
28
28
|
table_name=RDMExportingDataSubStageAttachment._meta.db_table,
|
|
29
29
|
field_name='exporting_data_sub_stage_id',
|
|
30
30
|
target_table=RDMExportingDataSubStage._meta.db_table,
|
|
31
|
-
on_delete='
|
|
31
|
+
on_delete='SET NULL',
|
|
32
32
|
)
|
|
33
33
|
|
|
34
34
|
RDMExportingDataSubStageEntity = apps.get_model(
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
# Generated by Django 3.2.24 on 2025-10-14 10:33
|
|
2
|
+
|
|
3
|
+
import django.db.models.deletion
|
|
4
|
+
from django.db import (
|
|
5
|
+
migrations,
|
|
6
|
+
models,
|
|
7
|
+
)
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
class Migration(migrations.Migration):
|
|
11
|
+
|
|
12
|
+
dependencies = [
|
|
13
|
+
('edu_rdm_integration_export_data_stage', '0003_fix_fk_constraints'),
|
|
14
|
+
]
|
|
15
|
+
|
|
16
|
+
operations = [
|
|
17
|
+
migrations.AlterField(
|
|
18
|
+
model_name='rdmexportingdatasubstageattachment',
|
|
19
|
+
name='exporting_data_sub_stage',
|
|
20
|
+
field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to='edu_rdm_integration_export_data_stage.rdmexportingdatasubstage', verbose_name='Подэтап выгрузки данных'),
|
|
21
|
+
),
|
|
22
|
+
]
|
|
@@ -69,6 +69,11 @@ class RDMExportingDataStageStatus(TitledModelEnum):
|
|
|
69
69
|
title='Завершено',
|
|
70
70
|
)
|
|
71
71
|
|
|
72
|
+
@classmethod
|
|
73
|
+
def get_choices(cls) -> list[tuple[str, str]]:
|
|
74
|
+
"""Возвращает список кортежей из ключей и ключей перечисления статусов."""
|
|
75
|
+
return [(key, key) for key in cls.get_model_enum_keys()]
|
|
76
|
+
|
|
72
77
|
class Meta:
|
|
73
78
|
db_table = 'rdm_exporting_data_stage_status'
|
|
74
79
|
verbose_name = 'Модель-перечисление статусов этапа выгрузки данных'
|
|
@@ -248,7 +253,9 @@ class RDMExportingDataSubStageAttachment(ReprStrPreModelMixin, BaseObjectModel):
|
|
|
248
253
|
exporting_data_sub_stage = ForeignKey(
|
|
249
254
|
to=RDMExportingDataSubStage,
|
|
250
255
|
verbose_name='Подэтап выгрузки данных',
|
|
251
|
-
|
|
256
|
+
null=True,
|
|
257
|
+
blank=True,
|
|
258
|
+
on_delete=SET_NULL,
|
|
252
259
|
)
|
|
253
260
|
|
|
254
261
|
# TODO PYTD-22 В зависимости от принятого решения по инструменту ограничения доступа к media-файлам, нужно будет
|
|
@@ -86,7 +86,7 @@ class BaseExportingDataProgressPack(BaseCommandProgressPack):
|
|
|
86
86
|
'header': 'Сущность',
|
|
87
87
|
'sortable': True,
|
|
88
88
|
'filter': ChoicesFilter(
|
|
89
|
-
choices=
|
|
89
|
+
choices=partial(RDMEntityEnum.get_choices),
|
|
90
90
|
parser=str,
|
|
91
91
|
lookup=lambda key: Q(entity=key) if key else Q(),
|
|
92
92
|
),
|
|
@@ -96,7 +96,7 @@ class BaseExportingDataProgressPack(BaseCommandProgressPack):
|
|
|
96
96
|
'header': 'Статус асинхронной задачи',
|
|
97
97
|
'sortable': True,
|
|
98
98
|
'filter': ChoicesFilter(
|
|
99
|
-
choices=
|
|
99
|
+
choices=partial(AsyncTaskStatus.get_choices),
|
|
100
100
|
parser=str,
|
|
101
101
|
lookup='task__status_id',
|
|
102
102
|
),
|
|
@@ -131,7 +131,7 @@ class BaseExportingDataProgressPack(BaseCommandProgressPack):
|
|
|
131
131
|
'header': 'Статус экспорта',
|
|
132
132
|
'sortable': True,
|
|
133
133
|
'filter': ChoicesFilter(
|
|
134
|
-
choices=
|
|
134
|
+
choices=partial(RDMExportingDataStageStatus.get_choices),
|
|
135
135
|
parser=str,
|
|
136
136
|
lookup=lambda key: Q(stage__status=key) if key else Q(),
|
|
137
137
|
),
|
|
@@ -1,8 +1,16 @@
|
|
|
1
|
+
import asyncio
|
|
1
2
|
from abc import (
|
|
2
3
|
ABCMeta,
|
|
3
4
|
abstractmethod,
|
|
4
5
|
)
|
|
6
|
+
from pathlib import (
|
|
7
|
+
Path,
|
|
8
|
+
)
|
|
9
|
+
from typing import (
|
|
10
|
+
TYPE_CHECKING,
|
|
11
|
+
)
|
|
5
12
|
|
|
13
|
+
import asyncpg
|
|
6
14
|
from django.conf import (
|
|
7
15
|
settings,
|
|
8
16
|
)
|
|
@@ -15,6 +23,12 @@ from educommon import (
|
|
|
15
23
|
)
|
|
16
24
|
|
|
17
25
|
|
|
26
|
+
if TYPE_CHECKING:
|
|
27
|
+
from asyncpg import (
|
|
28
|
+
Pool,
|
|
29
|
+
)
|
|
30
|
+
|
|
31
|
+
|
|
18
32
|
class BaseServiceOutdatedDataCleaner(metaclass=ABCMeta):
|
|
19
33
|
"""Базовый класс уборщика устаревших сервисных данных."""
|
|
20
34
|
|
|
@@ -112,8 +126,15 @@ class BaseServiceOutdatedDataCleaner(metaclass=ABCMeta):
|
|
|
112
126
|
"""Возвращает имя таблицы в базе данных."""
|
|
113
127
|
if cls.model is None:
|
|
114
128
|
raise NotImplementedError('Необходимо задать атрибут "model"')
|
|
129
|
+
|
|
115
130
|
return cls.model._meta.db_table
|
|
116
131
|
|
|
132
|
+
async def file_deletion_process(self, file_paths: list[str]):
|
|
133
|
+
"""Функция для удаления файлов, связанных с удалёнными устаревшими записями.
|
|
134
|
+
|
|
135
|
+
Очистка данных производится в таблицах системных моделей РВД.
|
|
136
|
+
"""
|
|
137
|
+
|
|
117
138
|
def get_orphan_reference_condition(
|
|
118
139
|
self,
|
|
119
140
|
reference_table: str,
|
|
@@ -148,67 +169,137 @@ class BaseServiceOutdatedDataCleaner(metaclass=ABCMeta):
|
|
|
148
169
|
)
|
|
149
170
|
"""
|
|
150
171
|
|
|
151
|
-
def
|
|
172
|
+
def get_chunk_bounded(self):
|
|
152
173
|
"""Возвращает границы чанков для текущей таблицы."""
|
|
153
|
-
|
|
174
|
+
get_chunk_bounded_sql = self.SELECT_RDM_CHUNK_BOUNDED_SQL.format(
|
|
154
175
|
table_name=self.get_table_name(),
|
|
155
|
-
chunk_size=settings.
|
|
156
|
-
)
|
|
157
|
-
with connection.cursor() as cursor:
|
|
158
|
-
cursor.execute(sql)
|
|
159
|
-
return cursor.fetchall()
|
|
160
|
-
|
|
161
|
-
def _log_query(self, sql: str):
|
|
162
|
-
"""Логирует SQL-запрос."""
|
|
163
|
-
try:
|
|
164
|
-
import sqlparse
|
|
165
|
-
sql = sqlparse.format(sql, reindent=True, strip_comments=True)
|
|
166
|
-
except ImportError:
|
|
167
|
-
pass
|
|
168
|
-
|
|
169
|
-
logger.info(
|
|
170
|
-
f'Запрос для удаления устаревших данных модели {self.get_table_name()}:\n{sql}\n'
|
|
176
|
+
chunk_size=settings.RDM_CLEANUP_MODELS_OUTDATED_DATA_CHUNK_SIZE,
|
|
171
177
|
)
|
|
172
178
|
|
|
173
|
-
def _execute_delete_sql(self, delete_sql: str) -> int:
|
|
174
|
-
"""Выполняет SQL-запрос на удаление (или только логирует в safe-режиме)."""
|
|
175
|
-
deleted = 0
|
|
176
179
|
if self._log_sql:
|
|
177
|
-
|
|
180
|
+
# Проверка на доступность sqlparse для форматирования
|
|
181
|
+
try:
|
|
182
|
+
import sqlparse
|
|
183
|
+
except ImportError:
|
|
184
|
+
sqlparse = None
|
|
178
185
|
|
|
179
|
-
|
|
186
|
+
if sqlparse:
|
|
187
|
+
# Форматирование кода
|
|
188
|
+
get_chunk_bounded_sql = sqlparse.format(
|
|
189
|
+
sql=get_chunk_bounded_sql,
|
|
190
|
+
reindent=True,
|
|
191
|
+
strip_comments=True,
|
|
192
|
+
)
|
|
180
193
|
logger.info(
|
|
181
|
-
f'
|
|
194
|
+
f'Запрос для получения границ чанков модели {self.get_table_name()}: \n{get_chunk_bounded_sql}\n'
|
|
182
195
|
)
|
|
183
|
-
return 0
|
|
184
|
-
else:
|
|
185
|
-
with connection.cursor() as cursor:
|
|
186
|
-
cursor.execute(delete_sql)
|
|
187
|
-
result = cursor.fetchone()
|
|
188
|
-
deleted = result[0] if result else 0
|
|
189
196
|
|
|
190
|
-
|
|
197
|
+
with connection.cursor() as cursor:
|
|
198
|
+
cursor.execute(get_chunk_bounded_sql)
|
|
199
|
+
result = cursor.fetchall()
|
|
191
200
|
|
|
192
|
-
|
|
193
|
-
|
|
201
|
+
return result
|
|
202
|
+
|
|
203
|
+
async def execute_query(self, pool: 'Pool', query: str):
|
|
204
|
+
"""Асинхронное выполнение запроса."""
|
|
205
|
+
async with pool.acquire() as conn:
|
|
206
|
+
try:
|
|
207
|
+
if self._safe:
|
|
208
|
+
logger.info(f'Запрос не будет выполнен, включен безопасный режим!\n')
|
|
209
|
+
|
|
210
|
+
if self._log_sql:
|
|
211
|
+
logger.info(f'{query}\n')
|
|
212
|
+
else:
|
|
213
|
+
result = await conn.fetch(query)
|
|
214
|
+
if not result:
|
|
215
|
+
return
|
|
216
|
+
|
|
217
|
+
if self._log_sql:
|
|
218
|
+
logger.info(f'При помощи запроса:\n{query}\n')
|
|
219
|
+
|
|
220
|
+
# Проверяем, что вернул запрос
|
|
221
|
+
if 'deleted_count' in result[0]:
|
|
222
|
+
deleted_count = result[0]['deleted_count']
|
|
223
|
+
self._deleted_count += deleted_count
|
|
224
|
+
logger.info(f'Было удалено записей: {deleted_count}')
|
|
225
|
+
else:
|
|
226
|
+
file_paths = [record['file_path'] for record in result if record.get('file_path')]
|
|
227
|
+
if file_paths:
|
|
228
|
+
await self.file_deletion_process(file_paths)
|
|
229
|
+
deleted_count = len(result)
|
|
230
|
+
self._deleted_count += deleted_count
|
|
231
|
+
logger.info(f'Было удалено записей с файлами: {deleted_count}')
|
|
232
|
+
|
|
233
|
+
except Exception as e:
|
|
234
|
+
logger.error(f'Ошибка при выполнении {query}\n{e}')
|
|
235
|
+
|
|
236
|
+
def prepare_queries(self, chunk_bounded: list[tuple[int, int, int]]) -> list[str]:
|
|
237
|
+
"""Формирование списка запросов для удаления устаревших данных."""
|
|
238
|
+
queries = []
|
|
194
239
|
conditions = self.get_merged_conditions()
|
|
195
240
|
|
|
196
|
-
# Разделяем по чанкам
|
|
197
|
-
chunk_bounded = self.get_chunk_bounds()
|
|
198
241
|
for chunk_number, first_id, last_id in chunk_bounded:
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
242
|
+
remove_outdated_data_sql = self.REMOVE_OUTDATED_DATA_SQL.format(
|
|
243
|
+
table_name=self.get_table_name(),
|
|
244
|
+
first_id=first_id,
|
|
245
|
+
last_id=last_id,
|
|
246
|
+
conditions=conditions,
|
|
247
|
+
)
|
|
248
|
+
|
|
249
|
+
queries.append(remove_outdated_data_sql)
|
|
250
|
+
|
|
251
|
+
return queries
|
|
208
252
|
|
|
209
|
-
|
|
210
|
-
|
|
253
|
+
async def execute_queries(self, queries: list[str]) -> None:
|
|
254
|
+
"""Асинхронное выполнение запросов."""
|
|
255
|
+
DB_SETTINGS = settings.DATABASES['default']
|
|
211
256
|
|
|
212
|
-
|
|
213
|
-
|
|
257
|
+
pool = await asyncpg.create_pool(
|
|
258
|
+
max_size=settings.RDM_CLEANUP_MODELS_OUTDATED_DATA_POOL_SIZE,
|
|
259
|
+
min_size=settings.RDM_CLEANUP_MODELS_OUTDATED_DATA_POOL_SIZE,
|
|
260
|
+
host=DB_SETTINGS['HOST'],
|
|
261
|
+
port=DB_SETTINGS['PORT'],
|
|
262
|
+
user=DB_SETTINGS['USER'],
|
|
263
|
+
password=DB_SETTINGS['PASSWORD'],
|
|
264
|
+
database=DB_SETTINGS['NAME'],
|
|
214
265
|
)
|
|
266
|
+
|
|
267
|
+
tasks = [self.execute_query(pool, query) for query in queries]
|
|
268
|
+
|
|
269
|
+
await asyncio.gather(*tasks)
|
|
270
|
+
|
|
271
|
+
def run(self):
|
|
272
|
+
"""Запуск очистки устаревших данных."""
|
|
273
|
+
chunk_bounded = self.get_chunk_bounded()
|
|
274
|
+
|
|
275
|
+
queries = self.prepare_queries(chunk_bounded=chunk_bounded)
|
|
276
|
+
|
|
277
|
+
if queries:
|
|
278
|
+
even_loop = asyncio.new_event_loop()
|
|
279
|
+
try:
|
|
280
|
+
even_loop.run_until_complete(self.execute_queries(queries=queries))
|
|
281
|
+
finally:
|
|
282
|
+
even_loop.close()
|
|
283
|
+
|
|
284
|
+
logger.info(f'Удалено записей модели {self.model.__name__}: {self._deleted_count}')
|
|
285
|
+
|
|
286
|
+
|
|
287
|
+
class ServiceFileCleaner:
|
|
288
|
+
"""Асинхронный сервис для безопасного удаления файлов из MEDIA_ROOT."""
|
|
289
|
+
|
|
290
|
+
@staticmethod
|
|
291
|
+
async def file_deletion_process(file_paths: list[str]) -> None:
|
|
292
|
+
"""Удаляет указанные файлы, считая пути относительными к MEDIA_ROOT."""
|
|
293
|
+
media_root = Path(settings.MEDIA_ROOT).resolve()
|
|
294
|
+
|
|
295
|
+
async def delete_file(path_str: str):
|
|
296
|
+
path = (media_root / path_str).resolve()
|
|
297
|
+
try:
|
|
298
|
+
exists = await asyncio.to_thread(path.exists)
|
|
299
|
+
if exists and await asyncio.to_thread(path.is_file):
|
|
300
|
+
await asyncio.to_thread(path.unlink)
|
|
301
|
+
|
|
302
|
+
except Exception as e:
|
|
303
|
+
logger.warning(f"Не удалось удалить {path}: {e}")
|
|
304
|
+
|
|
305
|
+
await asyncio.gather(*(delete_file(path) for path in file_paths))
|
|
@@ -25,6 +25,7 @@ from educommon.utils.seqtools import (
|
|
|
25
25
|
|
|
26
26
|
from .base import (
|
|
27
27
|
BaseServiceOutdatedDataCleaner,
|
|
28
|
+
ServiceFileCleaner,
|
|
28
29
|
)
|
|
29
30
|
from .consts import (
|
|
30
31
|
OLD_RDM_MODEL,
|
|
@@ -102,15 +103,35 @@ class ExportingDataStageCleaner(BaseServiceOutdatedDataCleaner):
|
|
|
102
103
|
return self.get_orphan_reference_condition(sub_stage_table, 'stage_id')
|
|
103
104
|
|
|
104
105
|
|
|
105
|
-
class ExportingDataSubStageAttachmentCleaner(BaseServiceOutdatedDataCleaner):
|
|
106
|
+
class ExportingDataSubStageAttachmentCleaner(ServiceFileCleaner, BaseServiceOutdatedDataCleaner):
|
|
106
107
|
"""Очистка вложений подэтапов выгрузки данных."""
|
|
107
108
|
|
|
108
109
|
model = RDMExportingDataSubStageAttachment
|
|
109
110
|
|
|
111
|
+
REMOVE_OUTDATED_DATA_SQL = """
|
|
112
|
+
WITH deleted_rows AS (
|
|
113
|
+
DELETE FROM {table_name}
|
|
114
|
+
WHERE id IN (
|
|
115
|
+
WITH tbl AS (
|
|
116
|
+
SELECT *
|
|
117
|
+
FROM {table_name}
|
|
118
|
+
WHERE id >= {first_id}
|
|
119
|
+
AND id <= {last_id}
|
|
120
|
+
)
|
|
121
|
+
SELECT tbl.id
|
|
122
|
+
FROM tbl
|
|
123
|
+
WHERE {conditions}
|
|
124
|
+
)
|
|
125
|
+
RETURNING attachment AS file_path
|
|
126
|
+
)
|
|
127
|
+
SELECT file_path FROM deleted_rows;
|
|
128
|
+
"""
|
|
129
|
+
|
|
110
130
|
def get_merged_conditions(self) -> str:
|
|
111
131
|
"""Формирует условие удаления для устаревших данных."""
|
|
112
132
|
sub_stage_table = ExportingDataSubStageCleaner.get_table_name()
|
|
113
133
|
conditions = [
|
|
134
|
+
'exporting_data_sub_stage_id IS NULL',
|
|
114
135
|
f'({self.get_status_condition(sub_stage_table, "id", "FINISHED", 7, "exporting_data_sub_stage_id")})',
|
|
115
136
|
f'({self.get_status_condition(sub_stage_table, "id", "FAILED",30, "exporting_data_sub_stage_id")})',
|
|
116
137
|
f'({self.get_orphan_reference_condition(sub_stage_table, "id", "exporting_data_sub_stage_id")})',
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: edu-rdm-integration
|
|
3
|
-
Version: 3.22.
|
|
3
|
+
Version: 3.22.3
|
|
4
4
|
Summary: Интеграция с Региональной витриной данных
|
|
5
5
|
Author-email: BARS Group <education_dev@bars.group>
|
|
6
6
|
Project-URL: Homepage, https://stash.bars-open.ru/projects/EDUBASE/repos/edu-rdm-integration/browse
|
|
@@ -65,7 +65,7 @@ edu_rdm_integration/rdm_entities/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRk
|
|
|
65
65
|
edu_rdm_integration/rdm_entities/apps.py,sha256=2rg5IqBjzpvNsjK-1lnrknxH002XYTDCcrEcARCV-ms,302
|
|
66
66
|
edu_rdm_integration/rdm_entities/entities.py,sha256=zfIVpXwiNWEeotiS17xZtijSXe3rx6utCFhZ18fGPhM,14850
|
|
67
67
|
edu_rdm_integration/rdm_entities/mixins.py,sha256=x-IMk7iZuLyvOsVvZBPHu9EbxuCq7D9g2PliuR6yK8A,3088
|
|
68
|
-
edu_rdm_integration/rdm_entities/models.py,sha256=
|
|
68
|
+
edu_rdm_integration/rdm_entities/models.py,sha256=7jJ5DUj1Xn-OEplIvyItxrcDmc0sp08aa3BOgk1guB0,6045
|
|
69
69
|
edu_rdm_integration/rdm_entities/utils.py,sha256=zpVmqcJVYjQSHot9ZrVufDeSECP9kosgTmB8ydFDG8w,1095
|
|
70
70
|
edu_rdm_integration/rdm_entities/migrations/0001_initial.py,sha256=TL8zkmtbFdwdKkFy8wn5SC7dpLBFlS6s8PBp1YBPAog,1664
|
|
71
71
|
edu_rdm_integration/rdm_entities/migrations/0002_rename_regionaldatamartentityenum_rdmentityenum.py,sha256=CJmQp5TPAL66DBzho01tnWDrCa1fOmEI4AdaS0UbAB0,510
|
|
@@ -73,7 +73,7 @@ edu_rdm_integration/rdm_entities/migrations/__init__.py,sha256=47DEQpj8HBSa-_TIm
|
|
|
73
73
|
edu_rdm_integration/rdm_models/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
74
74
|
edu_rdm_integration/rdm_models/apps.py,sha256=JJwHzCXYVe5cEhvDg61KkrLoRT_ZbJCIAWEM2lOQyHU,288
|
|
75
75
|
edu_rdm_integration/rdm_models/mixins.py,sha256=h-xo2KckTf2pgetggQgo77FMVShdWfgcKNBkz5ct7sI,2308
|
|
76
|
-
edu_rdm_integration/rdm_models/models.py,sha256=
|
|
76
|
+
edu_rdm_integration/rdm_models/models.py,sha256=FqJ72Fmx_OIZhBybu_cqxYLAnQ6-6oI8mnMbyjBZeaY,14067
|
|
77
77
|
edu_rdm_integration/rdm_models/utils.py,sha256=Xk0HEpFEGAndoAD2TdubK4SI_dW2BvchQ7UeMEfvpfQ,631
|
|
78
78
|
edu_rdm_integration/rdm_models/migrations/0001_initial.py,sha256=qXgObuG2nfOLEnGJBoBqmq30TXetOv21UZU4trMV7mQ,1529
|
|
79
79
|
edu_rdm_integration/rdm_models/migrations/0002_rename_regionaldatamartmodelenum_rdmmodelenum.py,sha256=hNTLriOc9r9WEVKahJURA3yXhZ3ivbwJJ_HaMC46PpI,451
|
|
@@ -86,7 +86,7 @@ edu_rdm_integration/stages/collect_data/consts.py,sha256=tzaK9oxzdMRq3oTEocPz4um
|
|
|
86
86
|
edu_rdm_integration/stages/collect_data/generators.py,sha256=azl0s_xJp6Mg2ARNLKd4o4ikVcarUN3ysb4xm8pYlyY,12434
|
|
87
87
|
edu_rdm_integration/stages/collect_data/helpers.py,sha256=xy8z9yJKEMjNUPNhrsRRtnYy6RVbwDoD5zSDAX7y_6U,5260
|
|
88
88
|
edu_rdm_integration/stages/collect_data/mixins.py,sha256=izioaiPC26BDODgi_Lhy33IaH207945tGjFnbFLMQyI,2072
|
|
89
|
-
edu_rdm_integration/stages/collect_data/models.py,sha256=
|
|
89
|
+
edu_rdm_integration/stages/collect_data/models.py,sha256=PkBTXIcqv69eqtd084bKad1FtkJF6t0q8t7fmzEQYeY,9700
|
|
90
90
|
edu_rdm_integration/stages/collect_data/operations.py,sha256=K-St1Avwq093VU1fKzYB19vHRthK_DX6nGPO6PYAipk,11883
|
|
91
91
|
edu_rdm_integration/stages/collect_data/tests.py,sha256=OqxCSdSAOQm92WcBQRF2s0o3VYf-N4e60xpJ1TGdKO8,5550
|
|
92
92
|
edu_rdm_integration/stages/collect_data/function_templates/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
@@ -159,7 +159,7 @@ edu_rdm_integration/stages/collect_data/migrations/0005_alter_rdmcollectingdatas
|
|
|
159
159
|
edu_rdm_integration/stages/collect_data/migrations/0006_fix_fk_constraints.py,sha256=H58FBhu43KHy1aw7whdA-1ZktIXIWKwp4Nx3tpLz2nI,2284
|
|
160
160
|
edu_rdm_integration/stages/collect_data/migrations/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
161
161
|
edu_rdm_integration/stages/collect_data/registry/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
162
|
-
edu_rdm_integration/stages/collect_data/registry/actions.py,sha256
|
|
162
|
+
edu_rdm_integration/stages/collect_data/registry/actions.py,sha256=-3l944Csa5HCbrPqSOVwkVquoTBaju6BRtGOxF-Mnxc,10171
|
|
163
163
|
edu_rdm_integration/stages/collect_data/registry/apps.py,sha256=K5f97YXKMmdM7m33qgQYvJjrA8_eGAJ4VWyuRjJ0gwQ,439
|
|
164
164
|
edu_rdm_integration/stages/collect_data/registry/ui.py,sha256=pw13DAASxqnX_E5D4RG9CywtnQKQeljXHief7mojVgk,8398
|
|
165
165
|
edu_rdm_integration/stages/collect_data/registry/templates/ui-js/collect-command-window.js,sha256=QfxVSAA0282-41K0XGtyPa9WPzpoX_uClke8pHdAzBo,3112
|
|
@@ -170,7 +170,7 @@ edu_rdm_integration/stages/export_data/consts.py,sha256=ZEi1kXMs-54KFKxkyGIQVwZ4
|
|
|
170
170
|
edu_rdm_integration/stages/export_data/generators.py,sha256=XsTGcKm0oDgE3fUVDxRMkNFriOeuPDBAWi32nx5ASuc,3974
|
|
171
171
|
edu_rdm_integration/stages/export_data/helpers.py,sha256=uBl85AWS-V0usHeLf7VeAsy0ywvqXL_zT1Kgho-MRTM,6537
|
|
172
172
|
edu_rdm_integration/stages/export_data/mixins.py,sha256=YCr5aNcZmDx07JIr-vMietQQ3sZ82caWc7SWOMfdSjY,1772
|
|
173
|
-
edu_rdm_integration/stages/export_data/models.py,sha256=
|
|
173
|
+
edu_rdm_integration/stages/export_data/models.py,sha256=u0gUHGw0qB9QvuhVMAvJ7Xyg5uGvZdKm7oS1UVaxr_g,12340
|
|
174
174
|
edu_rdm_integration/stages/export_data/operations.py,sha256=g8rj4TaFnkRkaBD4omckBTGwsTlYpuBydOwwxY9hEiI,13188
|
|
175
175
|
edu_rdm_integration/stages/export_data/strategies.py,sha256=93pRhUOLL_q4VbWTY3CtC5J8rgPPO2EYn0fJk_oLuLk,6520
|
|
176
176
|
edu_rdm_integration/stages/export_data/function_templates/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
@@ -212,10 +212,11 @@ edu_rdm_integration/stages/export_data/management/commands/export_entities_data.
|
|
|
212
212
|
edu_rdm_integration/stages/export_data/management/commands/export_latest_entities_data.py,sha256=eYjBs_tZxcUAIseCyvsy5Jk-8k9Gm3xrG2dCuWNnrEs,1163
|
|
213
213
|
edu_rdm_integration/stages/export_data/migrations/0001_initial.py,sha256=h7HIT-QkWONCzIergDp2c861Lw_-f9TZQ5FPIREeHTk,16864
|
|
214
214
|
edu_rdm_integration/stages/export_data/migrations/0002_auto_20250704_0810.py,sha256=yjGlH25Mke2_VAe188LEkWZ04Xy1qenai7D2zOA-ecY,1558
|
|
215
|
-
edu_rdm_integration/stages/export_data/migrations/0003_fix_fk_constraints.py,sha256=
|
|
215
|
+
edu_rdm_integration/stages/export_data/migrations/0003_fix_fk_constraints.py,sha256=nmQTeRL7CSwOzVhn54bVRt41CZ7nQBw58RZNxbkEQvU,2979
|
|
216
|
+
edu_rdm_integration/stages/export_data/migrations/0004_alter_rdmexportingdatasubstageattachment_exporting_data_sub_stage.py,sha256=noj8EFPfymcNO-41--EFyuL0nKN3qjARKXL0R5SaAAg,694
|
|
216
217
|
edu_rdm_integration/stages/export_data/migrations/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
217
218
|
edu_rdm_integration/stages/export_data/registry/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
218
|
-
edu_rdm_integration/stages/export_data/registry/actions.py,sha256=
|
|
219
|
+
edu_rdm_integration/stages/export_data/registry/actions.py,sha256=zJ1GEzyhoGGRCEGxErmepPtec0qNJtcDJJEaPtcATdA,10847
|
|
219
220
|
edu_rdm_integration/stages/export_data/registry/apps.py,sha256=71DtJQ2ULt8_3CnTu2VAfT5ABBrDNY1nKTmZ6UtvIpw,448
|
|
220
221
|
edu_rdm_integration/stages/export_data/registry/ui.py,sha256=0kWWfOmtTyN4SUO9dGUcEz0wgOPA111jHoWw3_LNPqI,6238
|
|
221
222
|
edu_rdm_integration/stages/export_data/registry/templates/ui-js/create-export-command-win.js,sha256=g0dpYsvd_6VfRU4nRv3tNK-0wtMND_VurQRT04ShJjk,1341
|
|
@@ -229,10 +230,10 @@ edu_rdm_integration/stages/service/model_outdated_data/managers.py,sha256=0LNjvy
|
|
|
229
230
|
edu_rdm_integration/stages/service/service_outdated_data/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
230
231
|
edu_rdm_integration/stages/service/service_outdated_data/managers.py,sha256=zKGQWIvv1go5OlBIkgBDLNyEKkkHvm0qKFfCatddd68,2715
|
|
231
232
|
edu_rdm_integration/stages/service/service_outdated_data/cleaners/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
232
|
-
edu_rdm_integration/stages/service/service_outdated_data/cleaners/base.py,sha256=
|
|
233
|
+
edu_rdm_integration/stages/service/service_outdated_data/cleaners/base.py,sha256=7h6eSex_BakFz3jQiJ7R552SLOb9CPn6Z8xG__Vm_xg,10479
|
|
233
234
|
edu_rdm_integration/stages/service/service_outdated_data/cleaners/collect_data.py,sha256=zol47C4RQFnHd_MNldOHTtEjzeWsIZji1nDzKVVkJRg,4894
|
|
234
235
|
edu_rdm_integration/stages/service/service_outdated_data/cleaners/consts.py,sha256=ONLW_NJFbnYU1CbDF4mzBXXqDt7fyKVfep3Qlf5sJOM,384
|
|
235
|
-
edu_rdm_integration/stages/service/service_outdated_data/cleaners/export_data.py,sha256=
|
|
236
|
+
edu_rdm_integration/stages/service/service_outdated_data/cleaners/export_data.py,sha256=A27cOnTgYzQA2_JUedB2SSm6y-g82ADoubKqLT71txw,7153
|
|
236
237
|
edu_rdm_integration/stages/service/service_outdated_data/cleaners/upload_data.py,sha256=a8oJ8Sg1eLZf1NemehREkJsvdtmyLwtOIuN5iXpzSb4,2846
|
|
237
238
|
edu_rdm_integration/stages/upload_data/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
238
239
|
edu_rdm_integration/stages/upload_data/apps.py,sha256=aFhVPK-65b35CGKoAeAgQ0mm3STaWtZg7rqk3eL-b7s,620
|
|
@@ -267,8 +268,8 @@ edu_rdm_integration/stages/upload_data/uploader_log/ui.py,sha256=mU3XA9zVKHGqzNk
|
|
|
267
268
|
edu_rdm_integration/stages/upload_data/uploader_log/migrations/0001_initial.py,sha256=r5oOB7DBK9-mfuqPAgjXUJY5-hEcmMdILCwDTpaLnBc,753
|
|
268
269
|
edu_rdm_integration/stages/upload_data/uploader_log/migrations/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
269
270
|
edu_rdm_integration/stages/upload_data/uploader_log/templates/ui-js/object-grid-buttons.js,sha256=2xyGe0wdVokM0RhpzRzcRvJPBkBmPe3SlZry4oP4Nzs,6201
|
|
270
|
-
edu_rdm_integration-3.22.
|
|
271
|
-
edu_rdm_integration-3.22.
|
|
272
|
-
edu_rdm_integration-3.22.
|
|
273
|
-
edu_rdm_integration-3.22.
|
|
274
|
-
edu_rdm_integration-3.22.
|
|
271
|
+
edu_rdm_integration-3.22.3.dist-info/licenses/LICENSE,sha256=uw43Gjjj-1vXWCItfSrNDpbejnOwZMrNerUh8oWbq8Q,3458
|
|
272
|
+
edu_rdm_integration-3.22.3.dist-info/METADATA,sha256=42vNdYWuZfturJDneq57CUgZDkOsRvxCeexFvGh6DMI,43714
|
|
273
|
+
edu_rdm_integration-3.22.3.dist-info/WHEEL,sha256=1tXe9gY0PYatrMPMDd6jXqjfpz_B-Wqm32CPfRC58XU,91
|
|
274
|
+
edu_rdm_integration-3.22.3.dist-info/top_level.txt,sha256=nRJV0O14UtNE-jGIYG03sohgFnZClvf57H5m6VBXe9Y,20
|
|
275
|
+
edu_rdm_integration-3.22.3.dist-info/RECORD,,
|
|
File without changes
|
{edu_rdm_integration-3.22.1.dist-info → edu_rdm_integration-3.22.3.dist-info}/licenses/LICENSE
RENAMED
|
File without changes
|
|
File without changes
|