edu-rdm-integration 3.19.1__py3-none-any.whl → 3.21.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- edu_rdm_integration/rdm_entities/mixins.py +1 -1
- edu_rdm_integration/rdm_models/mixins.py +1 -1
- edu_rdm_integration/stages/service/model_outdated_data/cleaners.py +382 -2
- {edu_rdm_integration-3.19.1.dist-info → edu_rdm_integration-3.21.0.dist-info}/METADATA +15 -2
- {edu_rdm_integration-3.19.1.dist-info → edu_rdm_integration-3.21.0.dist-info}/RECORD +8 -8
- {edu_rdm_integration-3.19.1.dist-info → edu_rdm_integration-3.21.0.dist-info}/WHEEL +0 -0
- {edu_rdm_integration-3.19.1.dist-info → edu_rdm_integration-3.21.0.dist-info}/licenses/LICENSE +0 -0
- {edu_rdm_integration-3.19.1.dist-info → edu_rdm_integration-3.21.0.dist-info}/top_level.txt +0 -0
|
@@ -1,13 +1,44 @@
|
|
|
1
|
+
import asyncio
|
|
1
2
|
from abc import (
|
|
2
3
|
ABCMeta,
|
|
3
|
-
abstractmethod,
|
|
4
4
|
)
|
|
5
5
|
from typing import (
|
|
6
6
|
TYPE_CHECKING,
|
|
7
|
+
Optional,
|
|
8
|
+
)
|
|
9
|
+
|
|
10
|
+
import asyncpg
|
|
11
|
+
from django.conf import (
|
|
12
|
+
settings,
|
|
13
|
+
)
|
|
14
|
+
from django.db import (
|
|
15
|
+
connection,
|
|
16
|
+
)
|
|
17
|
+
from django.db.models import (
|
|
18
|
+
ForeignKey,
|
|
19
|
+
)
|
|
20
|
+
|
|
21
|
+
from educommon import (
|
|
22
|
+
logger,
|
|
23
|
+
)
|
|
24
|
+
from m3_db_utils.models import (
|
|
25
|
+
FictiveForeignKeyField,
|
|
26
|
+
)
|
|
27
|
+
|
|
28
|
+
from edu_rdm_integration.rdm_models.models import (
|
|
29
|
+
BaseMainRDMModel,
|
|
30
|
+
RDMModelEnum,
|
|
31
|
+
)
|
|
32
|
+
from edu_rdm_integration.stages.export_data.models import (
|
|
33
|
+
RDMExportingDataSubStageStatus,
|
|
7
34
|
)
|
|
8
35
|
|
|
9
36
|
|
|
10
37
|
if TYPE_CHECKING:
|
|
38
|
+
from asyncpg import (
|
|
39
|
+
Pool,
|
|
40
|
+
)
|
|
41
|
+
|
|
11
42
|
from m3_db_utils.models import (
|
|
12
43
|
ModelEnumValue,
|
|
13
44
|
)
|
|
@@ -16,6 +47,78 @@ if TYPE_CHECKING:
|
|
|
16
47
|
class BaseModelOutdatedDataCleaner(metaclass=ABCMeta):
|
|
17
48
|
"""Базовый класс уборщика устаревших данных моделей РВД."""
|
|
18
49
|
|
|
50
|
+
# Запрос для разбиения таблицы на чанки и получения идентификаторов первых и последних записей
|
|
51
|
+
SELECT_RDM_CHUNK_BOUNDED_SQL = """
|
|
52
|
+
DO $$
|
|
53
|
+
DECLARE
|
|
54
|
+
chunk_size INT := {chunk_size};
|
|
55
|
+
last_id INT := 0;
|
|
56
|
+
first_id INT;
|
|
57
|
+
last_chunk_id INT;
|
|
58
|
+
BEGIN
|
|
59
|
+
-- Создаем временную таблицу без ON COMMIT DROP
|
|
60
|
+
DROP TABLE IF EXISTS rdm_chunk_bounds;
|
|
61
|
+
CREATE TEMP TABLE rdm_chunk_bounds (
|
|
62
|
+
chunk_number INT,
|
|
63
|
+
first_id INT,
|
|
64
|
+
last_id INT
|
|
65
|
+
);
|
|
66
|
+
|
|
67
|
+
DROP TABLE IF EXISTS tmp_chunk;
|
|
68
|
+
CREATE TEMP TABLE tmp_chunk (id INT) ON COMMIT DROP;
|
|
69
|
+
|
|
70
|
+
WHILE TRUE LOOP
|
|
71
|
+
TRUNCATE tmp_chunk;
|
|
72
|
+
|
|
73
|
+
INSERT INTO tmp_chunk (id)
|
|
74
|
+
SELECT id
|
|
75
|
+
FROM {table_name}
|
|
76
|
+
WHERE id > last_id
|
|
77
|
+
ORDER BY id
|
|
78
|
+
LIMIT chunk_size;
|
|
79
|
+
|
|
80
|
+
IF NOT FOUND THEN
|
|
81
|
+
EXIT;
|
|
82
|
+
END IF;
|
|
83
|
+
|
|
84
|
+
SELECT MIN(id), MAX(id)
|
|
85
|
+
INTO first_id, last_chunk_id
|
|
86
|
+
FROM tmp_chunk;
|
|
87
|
+
|
|
88
|
+
INSERT INTO rdm_chunk_bounds (chunk_number, first_id, last_id)
|
|
89
|
+
VALUES (
|
|
90
|
+
(SELECT COUNT(*) FROM rdm_chunk_bounds) + 1,
|
|
91
|
+
first_id,
|
|
92
|
+
last_chunk_id
|
|
93
|
+
);
|
|
94
|
+
|
|
95
|
+
last_id := last_chunk_id;
|
|
96
|
+
END LOOP;
|
|
97
|
+
END $$;
|
|
98
|
+
|
|
99
|
+
-- Теперь можно безопасно выбрать данные
|
|
100
|
+
SELECT * FROM rdm_chunk_bounds ORDER BY chunk_number;
|
|
101
|
+
"""
|
|
102
|
+
|
|
103
|
+
REMOVE_OUTDATED_DATA_SQL = """
|
|
104
|
+
WITH deleted_rows AS (
|
|
105
|
+
DELETE FROM {table_name}
|
|
106
|
+
WHERE id IN (
|
|
107
|
+
WITH tbl AS (
|
|
108
|
+
SELECT *
|
|
109
|
+
FROM {table_name}
|
|
110
|
+
WHERE id >= {first_id}
|
|
111
|
+
AND id <= {last_id}
|
|
112
|
+
)
|
|
113
|
+
SELECT tbl.id
|
|
114
|
+
FROM tbl
|
|
115
|
+
WHERE {conditions}
|
|
116
|
+
)
|
|
117
|
+
RETURNING id
|
|
118
|
+
)
|
|
119
|
+
SELECT COUNT(*) AS deleted_count FROM deleted_rows;
|
|
120
|
+
"""
|
|
121
|
+
|
|
19
122
|
def __init__(
|
|
20
123
|
self,
|
|
21
124
|
model_enum_value: 'ModelEnumValue',
|
|
@@ -27,9 +130,286 @@ class BaseModelOutdatedDataCleaner(metaclass=ABCMeta):
|
|
|
27
130
|
self._model_enum_value = model_enum_value
|
|
28
131
|
self._safe = safe
|
|
29
132
|
self._log_sql = log_sql
|
|
133
|
+
self._deleted_count = 0
|
|
30
134
|
|
|
31
135
|
super().__init__(*args, **kwargs)
|
|
32
136
|
|
|
33
|
-
|
|
137
|
+
def get_remove_empty_related_data_conditions(self) -> Optional[str]:
|
|
138
|
+
"""Формирование условий очистки данных, содержащих связи с моделями с отсутствующими связанными записями."""
|
|
139
|
+
remove_empty_related_data_conditions = None
|
|
140
|
+
|
|
141
|
+
rel_model_condition_template = """
|
|
142
|
+
NOT EXISTS (
|
|
143
|
+
SELECT 1 FROM {0} WHERE {0}.id = tbl.{1}
|
|
144
|
+
)
|
|
145
|
+
"""
|
|
146
|
+
|
|
147
|
+
rel_model_null_condition_template = """
|
|
148
|
+
tbl.{1} IS NOT NULL AND NOT EXISTS (
|
|
149
|
+
SELECT 1 FROM {0} WHERE {0}.id = tbl.{1}
|
|
150
|
+
)
|
|
151
|
+
"""
|
|
152
|
+
|
|
153
|
+
remove_empty_related_data_conditions_template = """
|
|
154
|
+
(
|
|
155
|
+
{}
|
|
156
|
+
)
|
|
157
|
+
-- Добавлено условие: удалить только записи старше 30 дней
|
|
158
|
+
AND tbl.modified <= NOW() - INTERVAL '30 days'
|
|
159
|
+
"""
|
|
160
|
+
|
|
161
|
+
model_label_enum_data = RDMModelEnum.get_model_label_enum_data()
|
|
162
|
+
related_fields = []
|
|
163
|
+
|
|
164
|
+
for field in self._model_enum_value.model._meta.concrete_fields:
|
|
165
|
+
if isinstance(field, FictiveForeignKeyField):
|
|
166
|
+
model_label = field.to
|
|
167
|
+
elif isinstance(field, ForeignKey):
|
|
168
|
+
model_label = field.related_model._meta.label
|
|
169
|
+
else:
|
|
170
|
+
continue
|
|
171
|
+
|
|
172
|
+
if model_label in model_label_enum_data:
|
|
173
|
+
table_name = model_label_enum_data[model_label].model._meta.db_table
|
|
174
|
+
related_fields.append((field.name, table_name, field.null))
|
|
175
|
+
|
|
176
|
+
if related_fields:
|
|
177
|
+
rel_model_conditions_list = []
|
|
178
|
+
|
|
179
|
+
for field_name, table_name, is_null in related_fields:
|
|
180
|
+
if is_null:
|
|
181
|
+
rel_model_conditions_list.append(
|
|
182
|
+
rel_model_null_condition_template.format(table_name, field_name)
|
|
183
|
+
)
|
|
184
|
+
else:
|
|
185
|
+
rel_model_conditions_list.append(rel_model_condition_template.format(table_name, field_name))
|
|
186
|
+
|
|
187
|
+
rel_model_conditions = ' OR '.join(rel_model_conditions_list)
|
|
188
|
+
|
|
189
|
+
remove_empty_related_data_conditions = remove_empty_related_data_conditions_template.format(
|
|
190
|
+
rel_model_conditions
|
|
191
|
+
)
|
|
192
|
+
|
|
193
|
+
return remove_empty_related_data_conditions
|
|
194
|
+
|
|
195
|
+
def get_remove_closed_periods_data_conditions(self) -> Optional[str]:
|
|
196
|
+
"""Формирование условия очистки данных по закрытым периодам обучения - учебным годам.
|
|
197
|
+
|
|
198
|
+
Очистка данных производится в таблицах моделей РВД. Производится обход по всем периодам обучения. Если с даты
|
|
199
|
+
окончания периода обучения прошло более 30 дней, то данные по модели РВД удаляются.
|
|
200
|
+
"""
|
|
201
|
+
|
|
202
|
+
def get_remove_data_by_deleted_records_conditions(self) -> Optional[str]:
|
|
203
|
+
"""Формирование условия очистки данных по удаленным записям в ЭШ.
|
|
204
|
+
|
|
205
|
+
Если данные в ЭШ были удалены, то после оповещения "Региональной витрины данных" об удалении того или иного
|
|
206
|
+
экземпляра сущности, данные могут быть удалены, т.к. больше не будут обновляться.
|
|
207
|
+
"""
|
|
208
|
+
remove_data_by_deleted_records_conditions_sql = None
|
|
209
|
+
|
|
210
|
+
if issubclass(self._model_enum_value.model, BaseMainRDMModel):
|
|
211
|
+
remove_data_by_deleted_records_conditions_sql = """
|
|
212
|
+
EXISTS (
|
|
213
|
+
SELECT 1
|
|
214
|
+
FROM rdm_exporting_data_sub_stage redss
|
|
215
|
+
WHERE redss.id = tbl.exporting_sub_stage_id
|
|
216
|
+
AND redss.status_id = 'FINISHED'
|
|
217
|
+
AND tbl.operation = 3
|
|
218
|
+
)
|
|
219
|
+
"""
|
|
220
|
+
|
|
221
|
+
return remove_data_by_deleted_records_conditions_sql
|
|
222
|
+
|
|
223
|
+
def get_reverse_relation_conditions(self) -> Optional[str]:
|
|
224
|
+
"""Формирование условий для удаления записей по связанным моделям.
|
|
225
|
+
|
|
226
|
+
Записи по связанным моделям удаляются по следующим условиям:
|
|
227
|
+
- если запись в таблице с обратной связью не найдена или была успешно выгружена в витрину.
|
|
228
|
+
"""
|
|
229
|
+
processed_model_fields = []
|
|
230
|
+
not_fished_exporting_data_statuses = ', '.join(
|
|
231
|
+
[f"'{status.key}'" for status in RDMExportingDataSubStageStatus.get_not_finished_statuses()]
|
|
232
|
+
)
|
|
233
|
+
conditions = []
|
|
234
|
+
main_model_condition_template = """
|
|
235
|
+
NOT EXISTS (
|
|
236
|
+
SELECT 1
|
|
237
|
+
FROM (SELECT DISTINCT exporting_sub_stage_id, {2} FROM {0}) {1}
|
|
238
|
+
LEFT JOIN (SELECT id, status_id FROM rdm_exporting_data_sub_stage) redss
|
|
239
|
+
ON redss.id = {1}.exporting_sub_stage_id
|
|
240
|
+
WHERE {1}.{2} = tbl.id AND redss.status_id IN ({3})
|
|
241
|
+
)
|
|
242
|
+
"""
|
|
243
|
+
enum_data = RDMModelEnum.get_enum_data()
|
|
244
|
+
for model_key, field_names in self._model_enum_value.reverse_relations.items():
|
|
245
|
+
model_enum_value = enum_data[model_key]
|
|
246
|
+
|
|
247
|
+
if issubclass(model_enum_value.model, BaseMainRDMModel):
|
|
248
|
+
for field_name in field_names:
|
|
249
|
+
if (model_enum_value.model._meta.db_table, field_name) not in processed_model_fields:
|
|
250
|
+
table_slug = ''.join(w[0] for w in model_enum_value.model._meta.db_table.split('_'))
|
|
251
|
+
|
|
252
|
+
conditions.append(
|
|
253
|
+
main_model_condition_template.format(
|
|
254
|
+
model_enum_value.model._meta.db_table,
|
|
255
|
+
table_slug,
|
|
256
|
+
field_name,
|
|
257
|
+
not_fished_exporting_data_statuses,
|
|
258
|
+
)
|
|
259
|
+
)
|
|
260
|
+
|
|
261
|
+
processed_model_fields.append((model_enum_value.model._meta.db_table, field_name))
|
|
262
|
+
|
|
263
|
+
return ' AND '.join(conditions) if conditions else None
|
|
264
|
+
|
|
265
|
+
def get_remove_data_by_finished_status_conditions(self) -> Optional[str]:
|
|
266
|
+
"""Формирование условия для удаления записей по выгруженным моделям."""
|
|
267
|
+
remove_data_by_finished_status_conditions_sql = None
|
|
268
|
+
|
|
269
|
+
if issubclass(self._model_enum_value.model, BaseMainRDMModel):
|
|
270
|
+
remove_data_by_finished_status_conditions_sql = """
|
|
271
|
+
EXISTS (
|
|
272
|
+
SELECT 1
|
|
273
|
+
FROM rdm_exporting_data_sub_stage redss
|
|
274
|
+
WHERE redss.id = tbl.exporting_sub_stage_id
|
|
275
|
+
AND redss.status_id = 'FINISHED'
|
|
276
|
+
)
|
|
277
|
+
"""
|
|
278
|
+
|
|
279
|
+
return remove_data_by_finished_status_conditions_sql
|
|
280
|
+
|
|
281
|
+
def get_merged_conditions(self) -> str:
|
|
282
|
+
"""Формирование общего условия для определения устаревших записей."""
|
|
283
|
+
remove_empty_related_data_conditions = self.get_remove_empty_related_data_conditions()
|
|
284
|
+
remove_closed_periods_data_conditions = self.get_remove_closed_periods_data_conditions()
|
|
285
|
+
remove_data_by_deleted_records_conditions = self.get_remove_data_by_deleted_records_conditions()
|
|
286
|
+
reverse_relation_conditions = self.get_reverse_relation_conditions()
|
|
287
|
+
remove_data_by_finished_status_conditions = self.get_remove_data_by_finished_status_conditions()
|
|
288
|
+
|
|
289
|
+
conditions = ''
|
|
290
|
+
main_conditions = []
|
|
291
|
+
if remove_empty_related_data_conditions:
|
|
292
|
+
main_conditions.append(f'({remove_empty_related_data_conditions})')
|
|
293
|
+
|
|
294
|
+
if remove_closed_periods_data_conditions:
|
|
295
|
+
main_conditions.append(f'({remove_closed_periods_data_conditions})')
|
|
296
|
+
|
|
297
|
+
if remove_data_by_deleted_records_conditions:
|
|
298
|
+
main_conditions.append(f'({remove_data_by_deleted_records_conditions})')
|
|
299
|
+
|
|
300
|
+
if main_conditions:
|
|
301
|
+
conditions = ' OR '.join(main_conditions)
|
|
302
|
+
|
|
303
|
+
if reverse_relation_conditions and remove_data_by_finished_status_conditions:
|
|
304
|
+
additional_conditions = f'({reverse_relation_conditions}) AND ({remove_data_by_finished_status_conditions})'
|
|
305
|
+
else:
|
|
306
|
+
additional_conditions = reverse_relation_conditions or remove_data_by_finished_status_conditions
|
|
307
|
+
|
|
308
|
+
if conditions and additional_conditions:
|
|
309
|
+
conditions = f'({conditions}) AND ({additional_conditions})'
|
|
310
|
+
else:
|
|
311
|
+
conditions = additional_conditions or conditions
|
|
312
|
+
|
|
313
|
+
return conditions
|
|
314
|
+
|
|
315
|
+
def get_chunk_bounded(self) -> list[tuple[int, int, int]]:
|
|
316
|
+
"""Получение идентификаторов-границ чанков для проверки и удаления устаревших данных."""
|
|
317
|
+
get_chunk_bounded_sql = self.SELECT_RDM_CHUNK_BOUNDED_SQL.format(
|
|
318
|
+
chunk_size=settings.RDM_CLEANUP_MODELS_OUTDATED_DATA_CHUNK_SIZE,
|
|
319
|
+
table_name=self._model_enum_value.model._meta.db_table,
|
|
320
|
+
)
|
|
321
|
+
|
|
322
|
+
if self._log_sql:
|
|
323
|
+
# Проверка на доступность sqlparse для форматирования
|
|
324
|
+
try:
|
|
325
|
+
import sqlparse
|
|
326
|
+
except ImportError:
|
|
327
|
+
sqlparse = None
|
|
328
|
+
|
|
329
|
+
if sqlparse:
|
|
330
|
+
# Форматирование кода
|
|
331
|
+
get_chunk_bounded_sql = sqlparse.format(
|
|
332
|
+
sql=get_chunk_bounded_sql,
|
|
333
|
+
reindent=True,
|
|
334
|
+
strip_comments=True,
|
|
335
|
+
)
|
|
336
|
+
logger.info(
|
|
337
|
+
f'Запрос для получения границ чанков модели {self._model_enum_value.key}: \n{get_chunk_bounded_sql}\n'
|
|
338
|
+
)
|
|
339
|
+
|
|
340
|
+
with connection.cursor() as cursor:
|
|
341
|
+
cursor.execute(get_chunk_bounded_sql)
|
|
342
|
+
result = cursor.fetchall()
|
|
343
|
+
|
|
344
|
+
return result
|
|
345
|
+
|
|
346
|
+
def prepare_queries(self, chunk_bounded: list[tuple[int, int, int]]) -> list[str]:
|
|
347
|
+
"""Формирование списка запросов для удаления устаревших данных."""
|
|
348
|
+
queries = []
|
|
349
|
+
conditions = self.get_merged_conditions()
|
|
350
|
+
|
|
351
|
+
for chunk_number, first_id, last_id in chunk_bounded:
|
|
352
|
+
remove_outdated_data_sql = self.REMOVE_OUTDATED_DATA_SQL.format(
|
|
353
|
+
table_name=self._model_enum_value.model._meta.db_table,
|
|
354
|
+
first_id=first_id,
|
|
355
|
+
last_id=last_id,
|
|
356
|
+
conditions=conditions,
|
|
357
|
+
)
|
|
358
|
+
|
|
359
|
+
queries.append(remove_outdated_data_sql)
|
|
360
|
+
|
|
361
|
+
return queries
|
|
362
|
+
|
|
363
|
+
async def execute_query(self, pool: 'Pool', query: str) -> str:
|
|
364
|
+
"""Асинхронное выполнение запроса."""
|
|
365
|
+
async with pool.acquire() as conn:
|
|
366
|
+
try:
|
|
367
|
+
if self._safe:
|
|
368
|
+
logger.info(f'Запрос не будет выполнен, включен безопасный режим!\n')
|
|
369
|
+
|
|
370
|
+
if self._log_sql:
|
|
371
|
+
logger.info(f'{query}\n')
|
|
372
|
+
else:
|
|
373
|
+
deleted_count = await conn.fetchval(query)
|
|
374
|
+
|
|
375
|
+
self._deleted_count += deleted_count
|
|
376
|
+
|
|
377
|
+
if self._log_sql:
|
|
378
|
+
logger.info(f'При помощи запроса:\n{query}\n')
|
|
379
|
+
|
|
380
|
+
logger.info(f'Было удалено записей: {deleted_count}\n')
|
|
381
|
+
except Exception as e:
|
|
382
|
+
logger.error(f'Ошибка при выполнении {query}\n{e}')
|
|
383
|
+
|
|
384
|
+
async def execute_queries(self, queries: list[str]) -> None:
|
|
385
|
+
"""Асинхронное выполнение запросов."""
|
|
386
|
+
DB_SETTINGS = settings.DATABASES['default']
|
|
387
|
+
|
|
388
|
+
pool = await asyncpg.create_pool(
|
|
389
|
+
max_size=settings.RDM_CLEANUP_MODELS_OUTDATED_DATA_POOL_SIZE,
|
|
390
|
+
min_size=settings.RDM_CLEANUP_MODELS_OUTDATED_DATA_POOL_SIZE,
|
|
391
|
+
host=DB_SETTINGS['HOST'],
|
|
392
|
+
port=DB_SETTINGS['PORT'],
|
|
393
|
+
user=DB_SETTINGS['USER'],
|
|
394
|
+
password=DB_SETTINGS['PASSWORD'],
|
|
395
|
+
database=DB_SETTINGS['NAME'],
|
|
396
|
+
)
|
|
397
|
+
|
|
398
|
+
tasks = [self.execute_query(pool, query) for query in queries]
|
|
399
|
+
|
|
400
|
+
await asyncio.gather(*tasks)
|
|
401
|
+
|
|
34
402
|
def run(self):
|
|
35
403
|
"""Запуск очистки устаревших данных."""
|
|
404
|
+
chunk_bounded = self.get_chunk_bounded()
|
|
405
|
+
|
|
406
|
+
queries = self.prepare_queries(chunk_bounded=chunk_bounded)
|
|
407
|
+
|
|
408
|
+
if queries:
|
|
409
|
+
even_loop = asyncio.new_event_loop()
|
|
410
|
+
try:
|
|
411
|
+
even_loop.run_until_complete(self.execute_queries(queries=queries))
|
|
412
|
+
finally:
|
|
413
|
+
even_loop.close()
|
|
414
|
+
|
|
415
|
+
logger.info(f'Удалено записей модели {self._model_enum_value.key}: {self._deleted_count}')
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: edu-rdm-integration
|
|
3
|
-
Version: 3.
|
|
3
|
+
Version: 3.21.0
|
|
4
4
|
Summary: Интеграция с Региональной витриной данных
|
|
5
5
|
Author-email: BARS Group <education_dev@bars.group>
|
|
6
6
|
Project-URL: Homepage, https://stash.bars-open.ru/projects/EDUBASE/repos/edu-rdm-integration/browse
|
|
@@ -30,7 +30,7 @@ Requires-Dist: asyncpg==0.23.0
|
|
|
30
30
|
Requires-Dist: educommon<4,>=3.25.0
|
|
31
31
|
Requires-Dist: edu-function-tools<1,>=0.2.0
|
|
32
32
|
Requires-Dist: m3-db-utils<1,>=0.7.0
|
|
33
|
-
Requires-Dist: m3-django-
|
|
33
|
+
Requires-Dist: m3-django-compatibility<2,>=1.12.0
|
|
34
34
|
Requires-Dist: m3-ui<2.3,>=2.2.122
|
|
35
35
|
Requires-Dist: uploader-client<1,>=0.3.0
|
|
36
36
|
Provides-Extra: dev
|
|
@@ -198,6 +198,8 @@ INSTALLED_APPS = (
|
|
|
198
198
|
('uploader_client', 'INSTALLATION_NAME'): '',
|
|
199
199
|
('uploader_client', 'INSTALLATION_ID'): '',
|
|
200
200
|
('rdm_cleanup_outdated_data', 'ENABLE_CLEANUP_MODELS_OUTDATED_DATA'): False,
|
|
201
|
+
('rdm_cleanup_outdated_data', 'CLEANUP_MODELS_OUTDATED_DATA_CHUNK_SIZE'): 10000,
|
|
202
|
+
('rdm_cleanup_outdated_data', 'CLEANUP_MODELS_OUTDATED_DATA_POOL_SIZE'): 10,
|
|
201
203
|
})
|
|
202
204
|
```
|
|
203
205
|
- Получение значений настроек из конфигурационного файла в settings.py:
|
|
@@ -313,6 +315,10 @@ INSTALLED_APPS = (
|
|
|
313
315
|
|
|
314
316
|
# Включить зачистку устаревших данных моделей РВД
|
|
315
317
|
RDM_ENABLE_CLEANUP_MODELS_OUTDATED_DATA = conf.get_bool('rdm_cleanup_outdated_data', 'ENABLE_CLEANUP_MODELS_OUTDATED_DATA')
|
|
318
|
+
# Размер чанка записей зачистки устаревших данных моделей РВД
|
|
319
|
+
RDM_CLEANUP_MODELS_OUTDATED_DATA_CHUNK_SIZE = conf.get_int('rdm_cleanup_outdated_data', 'CLEANUP_MODELS_OUTDATED_DATA_CHUNK_SIZE')
|
|
320
|
+
# Количество подключений к БД для зачистки устаревших данных моделей РВД
|
|
321
|
+
RDM_CLEANUP_MODELS_OUTDATED_DATA_POOL_SIZE = conf.get_int('rdm_cleanup_outdated_data', 'CLEANUP_MODELS_OUTDATED_DATA_POOL_SIZE')
|
|
316
322
|
```
|
|
317
323
|
|
|
318
324
|
Перечень настроек в settings.py указан в таблице ниже.
|
|
@@ -373,6 +379,9 @@ INSTALLED_APPS = (
|
|
|
373
379
|
| RDM_EXPORT_LOG_DIR | Директория логов экспорта данных, доступных для скачивания | |
|
|
374
380
|
| RDM_UPLOAD_LOG_DIR | Директория логов отправки данных в витрину, доступных для скачивания | |
|
|
375
381
|
| RDM_ENABLE_CLEANUP_MODELS_OUTDATED_DATA | Включение зачистки устаревших данных моделей РВД | False |
|
|
382
|
+
| RDM_CLEANUP_MODELS_OUTDATED_DATA_CHUNK_SIZE | Размер чанка записей зачистки устаревших данных моделей РВД | 10000 |
|
|
383
|
+
| RDM_CLEANUP_MODELS_OUTDATED_DATA_POOL_SIZE | Количество подключений к БД для зачистки устаревших данных моделей РВД | 10 |
|
|
384
|
+
|
|
376
385
|
|
|
377
386
|
- В дефолтный конфиг проекта необходимо добавить:
|
|
378
387
|
|
|
@@ -473,6 +482,10 @@ INSTALLED_APPS = (
|
|
|
473
482
|
[rdm_cleanup_outdated_data]
|
|
474
483
|
# Включить зачистку устаревших данных моделей РВД
|
|
475
484
|
ENABLE_CLEANUP_MODELS_OUTDATED_DATA = False
|
|
485
|
+
# Размер чанка записей проверки устаревших данных моделей РВД
|
|
486
|
+
CLEANUP_MODELS_OUTDATED_DATA_CHUNK_SIZE = 10000
|
|
487
|
+
# Количество подключений к БД для проверки устаревших данных моделей РВД
|
|
488
|
+
CLEANUP_MODELS_OUTDATED_DATA_POOL_SIZE = 10
|
|
476
489
|
```
|
|
477
490
|
На основе дефолтного конфига произвести конфигурирование приложений.
|
|
478
491
|
|
|
@@ -64,7 +64,7 @@ edu_rdm_integration/pipelines/transfer/templates/ui-js/transferred-entity-list.j
|
|
|
64
64
|
edu_rdm_integration/rdm_entities/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
65
65
|
edu_rdm_integration/rdm_entities/apps.py,sha256=2rg5IqBjzpvNsjK-1lnrknxH002XYTDCcrEcARCV-ms,302
|
|
66
66
|
edu_rdm_integration/rdm_entities/entities.py,sha256=zfIVpXwiNWEeotiS17xZtijSXe3rx6utCFhZ18fGPhM,14850
|
|
67
|
-
edu_rdm_integration/rdm_entities/mixins.py,sha256=
|
|
67
|
+
edu_rdm_integration/rdm_entities/mixins.py,sha256=x-IMk7iZuLyvOsVvZBPHu9EbxuCq7D9g2PliuR6yK8A,3088
|
|
68
68
|
edu_rdm_integration/rdm_entities/models.py,sha256=vGDQTS-iLnt-THRd89s24uzSXR4nBHLeEdWdPDxf2r4,5759
|
|
69
69
|
edu_rdm_integration/rdm_entities/utils.py,sha256=zpVmqcJVYjQSHot9ZrVufDeSECP9kosgTmB8ydFDG8w,1095
|
|
70
70
|
edu_rdm_integration/rdm_entities/migrations/0001_initial.py,sha256=TL8zkmtbFdwdKkFy8wn5SC7dpLBFlS6s8PBp1YBPAog,1664
|
|
@@ -72,7 +72,7 @@ edu_rdm_integration/rdm_entities/migrations/0002_rename_regionaldatamartentityen
|
|
|
72
72
|
edu_rdm_integration/rdm_entities/migrations/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
73
73
|
edu_rdm_integration/rdm_models/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
74
74
|
edu_rdm_integration/rdm_models/apps.py,sha256=JJwHzCXYVe5cEhvDg61KkrLoRT_ZbJCIAWEM2lOQyHU,288
|
|
75
|
-
edu_rdm_integration/rdm_models/mixins.py,sha256=
|
|
75
|
+
edu_rdm_integration/rdm_models/mixins.py,sha256=h-xo2KckTf2pgetggQgo77FMVShdWfgcKNBkz5ct7sI,2308
|
|
76
76
|
edu_rdm_integration/rdm_models/models.py,sha256=vLJkOGP0opHoWbt-0GfrPz1CkQL9H2BsJbhzuTNHNJg,13235
|
|
77
77
|
edu_rdm_integration/rdm_models/utils.py,sha256=Xk0HEpFEGAndoAD2TdubK4SI_dW2BvchQ7UeMEfvpfQ,631
|
|
78
78
|
edu_rdm_integration/rdm_models/migrations/0001_initial.py,sha256=qXgObuG2nfOLEnGJBoBqmq30TXetOv21UZU4trMV7mQ,1529
|
|
@@ -220,7 +220,7 @@ edu_rdm_integration/stages/service/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQe
|
|
|
220
220
|
edu_rdm_integration/stages/service/apps.py,sha256=lgCG4_kpwgfDWh6y-GNuUwz5SOjkP7oS8kkUyVUcNRg,648
|
|
221
221
|
edu_rdm_integration/stages/service/tasks.py,sha256=PPCtT6EpLkAKRczY0KIT6GeE9eBkv60fl2W6KFvCRqc,2302
|
|
222
222
|
edu_rdm_integration/stages/service/model_outdated_data/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
223
|
-
edu_rdm_integration/stages/service/model_outdated_data/cleaners.py,sha256=
|
|
223
|
+
edu_rdm_integration/stages/service/model_outdated_data/cleaners.py,sha256=enlEPTj5_elo-vQTXfq0ar8i1aiN4BzfZQLXNGBsZ68,16795
|
|
224
224
|
edu_rdm_integration/stages/service/model_outdated_data/managers.py,sha256=0LNjvycTtSMGsN37U-otlPL_vlYJKZXtNXxwkseK1wA,2353
|
|
225
225
|
edu_rdm_integration/stages/service/service_outdated_data/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
226
226
|
edu_rdm_integration/stages/upload_data/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
@@ -254,8 +254,8 @@ edu_rdm_integration/stages/upload_data/uploader_log/ui.py,sha256=mU3XA9zVKHGqzNk
|
|
|
254
254
|
edu_rdm_integration/stages/upload_data/uploader_log/migrations/0001_initial.py,sha256=r5oOB7DBK9-mfuqPAgjXUJY5-hEcmMdILCwDTpaLnBc,753
|
|
255
255
|
edu_rdm_integration/stages/upload_data/uploader_log/migrations/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
256
256
|
edu_rdm_integration/stages/upload_data/uploader_log/templates/ui-js/object-grid-buttons.js,sha256=2xyGe0wdVokM0RhpzRzcRvJPBkBmPe3SlZry4oP4Nzs,6201
|
|
257
|
-
edu_rdm_integration-3.
|
|
258
|
-
edu_rdm_integration-3.
|
|
259
|
-
edu_rdm_integration-3.
|
|
260
|
-
edu_rdm_integration-3.
|
|
261
|
-
edu_rdm_integration-3.
|
|
257
|
+
edu_rdm_integration-3.21.0.dist-info/licenses/LICENSE,sha256=uw43Gjjj-1vXWCItfSrNDpbejnOwZMrNerUh8oWbq8Q,3458
|
|
258
|
+
edu_rdm_integration-3.21.0.dist-info/METADATA,sha256=kCdgMvokM8bolCEYgWhQYIDpWFV3hkfWK4MXL65gsHA,42284
|
|
259
|
+
edu_rdm_integration-3.21.0.dist-info/WHEEL,sha256=1tXe9gY0PYatrMPMDd6jXqjfpz_B-Wqm32CPfRC58XU,91
|
|
260
|
+
edu_rdm_integration-3.21.0.dist-info/top_level.txt,sha256=nRJV0O14UtNE-jGIYG03sohgFnZClvf57H5m6VBXe9Y,20
|
|
261
|
+
edu_rdm_integration-3.21.0.dist-info/RECORD,,
|
|
File without changes
|
{edu_rdm_integration-3.19.1.dist-info → edu_rdm_integration-3.21.0.dist-info}/licenses/LICENSE
RENAMED
|
File without changes
|
|
File without changes
|