lysync 1.0.0__py3-none-any.whl

lysync/tasks.py

@@ -0,0 +1,453 @@
+"""Celery 任务（移植自 costexcel/tasks/export_task.py 并适配新模型）。
+
+与旧版差异：
+* 任务名统一加 ``lysync_`` 前缀，避免与主项目其它 app 冲突。
+* 队列名为 ``lysync_export_queue``。
+* 导出逻辑改为直接写入目标库（见 :mod:`lysync.export.engine`），移除 Excel/OSS。
+* ``init_periodic_tasks`` 在 app 的 ``ready()`` 中调用，初始化定时任务。
+"""
+
+from __future__ import annotations
+
+import datetime
+import time
+import uuid
+
+from celery import shared_task
+from celery.exceptions import SoftTimeLimitExceeded
+from django.conf import settings
+from django.db import close_old_connections
+from django.utils import timezone
+from django_celery_beat.models import CrontabSchedule, PeriodicTask
+from loguru import logger
+from sqlglot import exp
+from sqlglot import parse_one
+
+from lysync.export.engine import exec_sync_export_check_item, exec_sync_export_task_item
+from lysync.export.utils import (
+    add_limit_with_sqlglot,
+    fetch_data_from_temp_db,
+    format_datasource_error_message,
+    get_sql_context_for_ai,
+    grade_order,
+    render_sql_template,
+    sql_performance_grade,
+)
+from lysync.models import (
+    EXPLAIN_STATUS_CHECKING,
+    EXPLAIN_STATUS_FAILED,
+    EXPORT_RECORD_STATUS_CHECKING,
+    EXPORT_RECORD_STATUS_FAIL,
+    EXPORT_RECORD_STATUS_RUNNING,
+    EXPORT_RECORD_STATUS_STOPPED,
+    EXPORT_RECORD_STATUS_STOPPING,
+    EXPORT_RECORD_STATUS_SUCCESS,
+    EXPORT_RECORD_STATUS_WAIT,
+    SyncDataSource,
+    SyncExportRecord,
+    SyncExportTask,
+)
+
+EXPORT_TASK_LOCK_PREFIX = "lysync_export_task_lock_"
+QUEUE_NAME = "lysync_export_queue"
+
+
+def _patch_logger(request_id=None):
+    try:
+        from util.middleware import get_request_id
+
+        return logger.patch(lambda record: record["extra"].update(**get_request_id()))
+    except Exception:
+        return logger
+
+
+# ============================ 定时任务初始化 ============================
+
+
+def init_periodic_tasks():
+    """初始化周期性任务（在 app ready() 时调用）。"""
+    try:
+        try:
+            schedule, _ = CrontabSchedule.objects.get_or_create(
+                minute="1", hour="1", day_of_week="*", day_of_month="*", month_of_year="*"
+            )
+        except CrontabSchedule.MultipleObjectsReturned:
+            schedule = CrontabSchedule.objects.filter(
+                minute="1", hour="1", day_of_week="*", day_of_month="*", month_of_year="*"
+            ).first()
+
+        PeriodicTask.objects.get_or_create(
+            name="lysync导出数据", task="lysync_export_data", defaults={"crontab": schedule}
+        )
+    except Exception as e:
+        logger.exception(f"初始化导出定时任务失败: {e}")
+
+    try:
+        try:
+            schedule, _ = CrontabSchedule.objects.get_or_create(
+                minute="*", hour="*", day_of_week="*", day_of_month="*", month_of_year="*"
+            )
+        except CrontabSchedule.MultipleObjectsReturned:
+            schedule = CrontabSchedule.objects.filter(
+                minute="*", hour="*", day_of_week="*", day_of_month="*", month_of_year="*"
+            ).first()
+
+        PeriodicTask.objects.get_or_create(
+            name="lysync校验中断任务", task="lysync_check_export_task_interrupt", defaults={"crontab": schedule}
+        )
+    except Exception as e:
+        logger.exception(f"初始化校验中断定时任务失败: {e}")
+
+
+# ============================ 中断校验 ============================
+
+
+@shared_task(name="lysync_check_export_task_interrupt")
+def check_export_task_interrupt():
+    from util.lock import RedisLock
+
+    close_old_connections()
+    logger.info("开始校验中断任务")
+    query = SyncExportTask.objects.filter(last_status=EXPORT_RECORD_STATUS_RUNNING, is_active=True)
+    for export_task in query:
+        if not RedisLock.check_lock_pattern(f"{EXPORT_TASK_LOCK_PREFIX}{export_task.id}"):
+            SyncExportTask.objects.filter(
+                id=export_task.id,
+                last_status=EXPORT_RECORD_STATUS_RUNNING,
+                update_time=export_task.update_time,
+            ).update(
+                last_status=EXPORT_RECORD_STATUS_FAIL,
+                last_message="任务被中断",
+                update_time=datetime.datetime.now(),
+            )
+
+
+# ============================ 导出主流程 ============================
+
+
+@shared_task(name="lysync_export_data")
+def exec_export_task(export_task_id=None, export_all=True):
+    request_id = uuid.uuid4().hex
+    logger.info(f"[{request_id}]开始执行导出任务: export_task_id:{export_task_id} export_all:{export_all}")
+    close_old_connections()
+    query_export_task = SyncExportTask.objects.filter(is_active=True).order_by("model_type__sorted")
+    now = datetime.datetime.now()
+    if export_task_id:
+        query_export_task = query_export_task.filter(id=export_task_id)
+    if not query_export_task.exists():
+        logger.info(f"[{request_id}]没有待处理的导出任务")
+        return
+    export_task_id_list = list(query_export_task.values_list("id", flat=True))
+    exec_export_task_queue.apply_async(
+        kwargs={
+            "request_id": request_id,
+            "export_task_id_list": export_task_id_list,
+            "export_all": export_all,
+            "now": now,
+            "index": 0,
+        },
+        queue=QUEUE_NAME,
+    )
+
+
+@shared_task(name="lysync_export_task_queue")
+def exec_export_task_queue(export_task_id_list, export_all, now, index=0, request_id=None):
+    try:
+        close_old_connections()
+        exec_export_task_queue_func(
+            export_task_id_list, export_all=export_all, now=now, index=index, request_id=request_id
+        )
+    except SoftTimeLimitExceeded:
+        logger.error(f"[{request_id}]任务超时: export_task_id_list={export_task_id_list}")
+        SyncExportTask.objects.filter(id=export_task_id_list[0], last_status=EXPORT_RECORD_STATUS_RUNNING).update(
+            last_status=EXPORT_RECORD_STATUS_FAIL, last_message="任务执行超时"
+        )
+
+
+def exec_export_task_queue_func(export_task_id_list, export_all, now, index, request_id=None):
+    from util.lock import RedisLock
+
+    if not export_task_id_list:
+        return
+    lock = None
+    try:
+        lock = RedisLock(f"{EXPORT_TASK_LOCK_PREFIX}{export_task_id_list[0]}", blocking=False)
+        if not lock.acquire():
+            logger.error(f"[{request_id}][SKIP]任务执行中:id:{export_task_id_list[0]}")
+            return
+        res = SyncExportTask.objects.filter(id=export_task_id_list[0], is_active=True).update(
+            last_status=EXPORT_RECORD_STATUS_RUNNING, update_time=timezone.now(), last_message=""
+        )
+        export_task = SyncExportTask.objects.filter(id=export_task_id_list[0], is_active=True).first()
+        if not res:
+            logger.error(f"[{request_id}]未找到导出任务: {export_task_id_list[0]}")
+            if len(export_task_id_list) > 1:
+                exec_export_task_queue.apply_async(
+                    kwargs={
+                        "request_id": request_id,
+                        "export_task_id_list": export_task_id_list[1:],
+                        "export_all": export_all,
+                        "now": now,
+                        "index": index + 1,
+                    },
+                    queue=QUEUE_NAME,
+                )
+            return
+        success = False
+        stop_task = False
+        start_new_task = False
+        message = ""
+        try:
+            success, message, has_more = exec_sync_export_task_item(export_task, request_id=request_id)
+            if export_all and has_more:
+                export_task = SyncExportTask.objects.filter(
+                    id=export_task_id_list[0], last_status=EXPORT_RECORD_STATUS_RUNNING
+                ).first()
+                if not export_task:
+                    stop_task = True
+                    logger.error(f"[{request_id}]任务已终止: {export_task_id_list}")
+                else:
+                    start_new_task = True
+        finally:
+            if stop_task:
+                SyncExportTask.objects.filter(
+                    id=export_task_id_list[0], last_status=EXPORT_RECORD_STATUS_STOPPING
+                ).update(last_status=EXPORT_RECORD_STATUS_STOPPED, last_message="已停止", update_time=timezone.now())
+            elif success:
+                SyncExportTask.objects.filter(
+                    id=export_task_id_list[0], last_status=EXPORT_RECORD_STATUS_RUNNING
+                ).update(last_status=EXPORT_RECORD_STATUS_SUCCESS, last_message=message, update_time=timezone.now())
+            else:
+                SyncExportTask.objects.filter(
+                    id=export_task_id_list[0], last_status=EXPORT_RECORD_STATUS_RUNNING
+                ).update(last_status=EXPORT_RECORD_STATUS_FAIL, last_message=message, update_time=timezone.now())
+
+            if start_new_task:
+                logger.info(f"[{request_id}]继续执行本任务的后续数据: id={export_task.id}, name={export_task.name}")
+                exec_export_task_queue.apply_async(
+                    kwargs={
+                        "request_id": request_id,
+                        "export_task_id_list": export_task_id_list,
+                        "export_all": export_all,
+                        "now": now,
+                        "index": index + 1,
+                    },
+                    queue=QUEUE_NAME,
+                )
+            else:
+                exec_export_task_check.apply_async(kwargs={"export_task_id": export_task_id_list[0]}, queue=QUEUE_NAME)
+                if len(export_task_id_list) > 1:
+                    logger.info(f"[{request_id}]继续执行下一个任务:{export_task_id_list[1:]}")
+                    exec_export_task_queue.apply_async(
+                        kwargs={
+                            "request_id": request_id,
+                            "export_task_id_list": export_task_id_list[1:],
+                            "export_all": export_all,
+                            "now": now,
+                            "index": index + 1,
+                        },
+                        queue=QUEUE_NAME,
+                    )
+                else:
+                    logger.info(f"[{request_id}]无其他任务，本次请求完成")
+    finally:
+        if lock:
+            lock.release()
+
+
+# ============================ 增量补录（check） ============================
+
+
+@shared_task(name="lysync_export_task_check")
+def exec_export_task_check(export_task_id=None):
+    close_old_connections()
+    logger.info("开始执行导出数据补录任务")
+    query_export_task = SyncExportTask.objects.filter(is_active=True)
+    now = datetime.datetime.now()
+    if export_task_id:
+        query_export_task = query_export_task.filter(id=export_task_id)
+    if not query_export_task.exists():
+        logger.info("没有待处理的任务")
+        return
+    for export_task in query_export_task:
+        exec_export_task_check_one_task.apply_async(
+            kwargs={"export_task_id": export_task.id, "now": now}, queue=QUEUE_NAME
+        )
+
+
+@shared_task(name="lysync_export_task_check_one_task")
+def exec_export_task_check_one_task(export_task_id, now):
+    export_record_query = SyncExportRecord.objects.filter(
+        export_task_id=export_task_id,
+        is_active=True,
+        check_status__in=[EXPORT_RECORD_STATUS_WAIT, EXPORT_RECORD_STATUS_CHECKING],
+    )
+    if not export_record_query.exists():
+        logger.info(f"未找到导出记录: {export_task_id}")
+        return
+    for index, export_record in enumerate(export_record_query):
+        exec_export_task_check_one_record.apply_async(
+            kwargs={
+                "export_task_id": export_task_id,
+                "export_record_id": export_record.id,
+                "now": now,
+                "index": index,
+            },
+            queue=QUEUE_NAME,
+        )
+
+
+@shared_task(name="lysync_export_task_check_one_record")
+def exec_export_task_check_one_record(export_task_id, export_record_id, now, index=0):
+    try:
+        close_old_connections()
+        export_record = SyncExportRecord.objects.filter(id=export_record_id, is_active=True).first()
+        if not export_record:
+            return
+        exec_sync_export_check_item(export_record)
+    except SoftTimeLimitExceeded:
+        logger.error(f"导出数据补录_单个任务超时: id={export_record_id}")
+        SyncExportRecord.objects.filter(id=export_record_id, check_status=EXPORT_RECORD_STATUS_CHECKING).update(
+            check_status=EXPORT_RECORD_STATUS_WAIT, last_check_message="任务执行超时"
+        )
+
+
+# ============================ 数据源连接校验 ============================
+
+
+@shared_task(name="lysync_check_datasource_connect", ignore_result=True)
+def check_datasource_connect(datasource_id):
+    logger.info(f"开始检查数据源连接: {datasource_id}")
+    for i in range(3):
+        obj = SyncDataSource.objects.filter(
+            id=datasource_id,
+            source_type=SyncDataSource.SOURCE_TYPE_DB,
+            is_active=True,
+        ).first()
+        if not obj:
+            logger.error(f"未找到数据源: {datasource_id}，第{i+1}次尝试")
+            time.sleep(3)
+        else:
+            break
+    if obj.db_connect_status != SyncDataSource.CONNECT_STATUS_CHECKING:
+        logger.info(f"数据源 {datasource_id}未启动检查")
+        return
+    try:
+        fetch_data_from_temp_db(
+            obj.db_type,
+            obj.db_host,
+            obj.db_port,
+            obj.db_user,
+            obj.db_password,
+            obj.db_name,
+            obj.db_options,
+            "select 1",
+        )
+    except Exception as e:
+        error_msg = format_datasource_error_message(str(e), obj.db_name, obj.db_host, obj.db_port)
+        obj.db_connect_status = SyncDataSource.CONNECT_STATUS_FAIL
+        obj.db_connect_message = error_msg
+        obj.save(update_fields=["db_connect_status", "db_connect_message"])
+        return
+    obj.db_connect_status = SyncDataSource.CONNECT_STATUS_SUCCESS
+    obj.db_connect_message = "连接成功"
+    obj.save(update_fields=["db_connect_status", "db_connect_message"])
+
+
+# ============================ SQL 性能分析 / AI 优化 ============================
+
+
+@shared_task(name="lysync_explain_item", ignore_result=True)
+def explain_item(export_task_id):
+    export_task = SyncExportTask.objects.filter(id=export_task_id, is_active=True).first()
+    if not export_task:
+        return
+    export_task.explain_status = EXPLAIN_STATUS_CHECKING
+    export_task.explain_json = None
+    export_task.save(update_fields=["explain_status", "explain_json", "update_time"])
+    try:
+        data_source = export_task.data_source
+        if data_source.db_type == "tsql":
+            export_task.explain_status = EXPLAIN_STATUS_FAILED
+            export_task.explain_json = {"error_message": "不支持 SQL Server"}
+            export_task.save(update_fields=["explain_status", "explain_json", "update_time"])
+            return
+        sql = export_task.sql
+        # SQL 现在是 jinja2 模板，先渲染成可解析的 SQL 再做 EXPLAIN
+        rendered_sql = render_sql_template(
+            sql, start_time=datetime.datetime(2020, 1, 1), end_time=datetime.datetime(2020, 1, 2)
+        )
+        parsed = parse_one(rendered_sql, dialect=data_source.db_type)
+        if not isinstance(parsed, exp.Select):
+            export_task.explain_status = EXPLAIN_STATUS_FAILED
+            export_task.explain_json = {"error_message": "外层应为SELECT语句"}
+            export_task.save(update_fields=["explain_status", "explain_json", "update_time"])
+            return
+        if settings.EXPORT_DATA_LIMIT:
+            rendered_sql = add_limit_with_sqlglot(data_source.db_type, rendered_sql, settings.EXPORT_DATA_LIMIT)
+
+        try:
+            explain_sql = f"EXPLAIN FORMAT=TRADITIONAL {rendered_sql}"
+            explain_res = fetch_data_from_temp_db(
+                data_source.db_type,
+                data_source.db_host,
+                data_source.db_port,
+                data_source.db_user,
+                data_source.db_password,
+                data_source.db_name,
+                data_source.db_options,
+                explain_sql,
+            )
+        except Exception:
+            explain_sql = f"EXPLAIN {rendered_sql}"
+            explain_res = fetch_data_from_temp_db(
+                data_source.db_type,
+                data_source.db_host,
+                data_source.db_port,
+                data_source.db_user,
+                data_source.db_password,
+                data_source.db_name,
+                data_source.db_options,
+                explain_sql,
+            )
+
+        worst_index = 0
+        for item in explain_res:
+            item["p"] = sql_performance_grade(item)
+            order = grade_order.index(item["p"])
+            worst_index = max(worst_index, order)
+        export_task.explain_status = grade_order[worst_index]
+        export_task.explain_json = {"explain": explain_res}
+        export_task.save(update_fields=["explain_status", "explain_json", "update_time"])
+    except Exception as e:
+        logger.exception(f"explain_item执行失败: {e}")
+        export_task.explain_status = EXPLAIN_STATUS_FAILED
+        export_task.explain_json = {"error_message": str(e)}
+        export_task.save(update_fields=["explain_status", "explain_json", "update_time"])
+
+
+@shared_task(name="lysync_ai_sql_optimization_task")
+def ai_sql_optimization_task(export_task_id):
+    logger.info("开始执行：lysync_ai_sql_optimization_task")
+    export_task = SyncExportTask.objects.filter(id=export_task_id).first()
+    if not export_task:
+        raise Exception("未找到任务")
+    sql = export_task.sql
+    data_source = export_task.data_source
+    if data_source.db_type == "tsql":
+        return "# 不支持 SQL Server"
+    context = get_sql_context_for_ai(
+        sql,
+        data_source.db_type,
+        data_source.db_host,
+        data_source.db_port,
+        data_source.db_user,
+        data_source.db_password,
+        data_source.db_name,
+        data_source.db_options,
+    )
+    try:
+        from dify import call as dify_call
+    except Exception:
+        return "# 未配置 dify 调用（宿主项目需提供 dify.call）"
+    return dify_call(context)

lysync/tests.py

@@ -0,0 +1,3 @@
+from django.test import TestCase
+
+# Create your tests here.

lysync/urls.py

@@ -0,0 +1,31 @@
+#!/usr/bin/env python
+# encoding: utf-8
+
+"""lysync 导出功能 URL 配置。
+
+宿主项目可在根 urlconf 中通过 ``path("lysync/", include("lysync.urls"))`` 引入。
+"""
+from django.urls import re_path
+
+from lysync import views
+
+urlpatterns = [
+    re_path(r"^create_export_task$", views.create_export_task),
+    re_path(r"^update_export_task$", views.update_export_task),
+    re_path(r"^recover_export_task$", views.recover_export_task),
+    re_path(r"^exec_task_immediately$", views.exec_task_immediately),
+    re_path(r"^reset_task_last_export$", views.reset_task_last_export),
+    re_path(r"^reset_task_status$", views.reset_task_status),
+    re_path(r"^update_sync_data_source$", views.update_sync_data_source),
+    re_path(r"^get_collect_funcs$", views.get_collect_funcs),
+    re_path(r"^save_sync_model_type$", views.save_sync_model_type),
+    re_path(r"^get_process_funcs$", views.get_process_funcs),
+    re_path(r"^get_content_types$", views.get_content_types),
+    re_path(r"^get_model_fields$", views.get_model_fields),
+    re_path(r"^query_raw_data$", views.query_raw_data),
+    re_path(r"^check_explains$", views.check_explains),
+    re_path(r"^data_source_check_connect$", views.data_source_check_connect),
+    re_path(r"^stop_export_task$", views.stop_export_task),
+    re_path(r"^ai_sql_optimization$", views.ai_sql_optimization),
+    re_path(r"^get_ai_sql_optimization_result$", views.get_ai_sql_optimization_result),
+]

lysync/utils.py

@@ -0,0 +1,22 @@
+import datetime
+
+from django.conf import settings
+from django.utils import timezone
+
+
+def get_queue_name():
+    return getattr(settings, "QUEUE_QUEUE_EXPORT", "celery")
+
+
+def local_today():
+    """返回“今天”的本地日期，兼容宿主项目 ``USE_TZ=True/False``。
+
+    当宿主项目 ``USE_TZ=False`` 时，``timezone.now()`` 返回 naive datetime，
+    此时调用 ``timezone.localdate()`` 会抛
+    ``ValueError: localdate() cannot be applied to a naive datetime``。
+    这里做兼容：检测到 naive 时直接用 ``datetime.date.today()``，
+    避免该异常破坏 SQL 模板渲染 / 任务推进逻辑。
+    """
+    if timezone.is_naive(timezone.now()):
+        return datetime.date.today()
+    return timezone.localdate()