lysync 1.0.0__py3-none-any.whl

lysync/models.py

@@ -0,0 +1,383 @@
+from django.apps import apps
+from django.db.models import CASCADE
+from util import base_model_fields as models
+from util.basemodel import BaseModel
+
+
+# process_list 处理顺序为 SyncModelType -> SyncModelField -> SyncExportTask
+class SyncModelType(BaseModel):
+    name = models.CharField(max_length=50, verbose_name="模型名称", db_index=True)
+    app_label = models.CharField(max_length=100, verbose_name="应用标签", db_index=True)
+    model = models.CharField(max_length=100, verbose_name="模型名", db_index=True)
+    sorted = models.IntegerField(
+        default=0, db_index=True, is_default_query=False, is_base_query=False, verbose_name="数据处理排序"
+    )
+    process_list = models.JSONField(
+        verbose_name="处理函数列表", is_default_query=False, is_base_query=False, default=list
+    )
+
+    def get_target_model(self):
+        """根据 ``app_label`` / ``model`` 字符串解析出目标 Django 模型类。"""
+        return apps.get_model(self.app_label, self.model)
+
+    class Meta:
+        verbose_name = "数据模型"
+        ordering = ["sorted"]
+        order_json = ["id", "sorted"]
+        pageinfo = True
+
+
+class SyncModelField(BaseModel):
+    IMPORT_TRUE = True
+    IMPORT_FALSE = False
+    IMPORT_CHOICES = ((IMPORT_TRUE, "导入"), (IMPORT_FALSE, "非导入"))
+    name = models.CharField(max_length=50, verbose_name="字段名", db_index=True)
+    field_name = models.CharField(max_length=50, verbose_name="字段名", db_index=True)
+    model_type = models.ForeignKey(SyncModelType, on_delete=CASCADE, verbose_name="数据模型")
+    is_unique = models.BooleanField(
+        default=False, verbose_name="是否唯一字段", help_text="可以配置多个字段，用于唯一标识数据行"
+    )
+    is_datetime_index = models.BooleanField(
+        default=False, verbose_name="是否日期时间索引", help_text="只能配置一个字段，用户数据同步时确定时间范围"
+    )
+    process_list = models.JSONField(
+        verbose_name="处理函数列表", is_default_query=False, is_base_query=False, default=list
+    )
+    is_import_column = models.BooleanField(default=True, choices=IMPORT_CHOICES, verbose_name="是否为导入列")
+
+    class Meta:
+        verbose_name = "数据字段"
+        order_json = ["create_time", "name"]
+        list_json = {
+            "model_type__name": ("数据类型", "r"),
+        }
+        pageinfo = True
+
+
+class SyncDataSource(BaseModel):
+    DB_TYPE_MYSQL = "mysql"
+    DB_TYPE_TSQL = "tsql"
+    DB_TYPE_CHOICES = (
+        (DB_TYPE_MYSQL, "MySQL"),
+        (DB_TYPE_TSQL, "SQL Server"),
+    )
+    SOURCE_TYPE_DB = "db"
+    SOURCE_TYPE_CUSTOM = "custom"
+    SOURCE_TYPE_CHOICES = (
+        (SOURCE_TYPE_DB, "数据库"),
+        (SOURCE_TYPE_CUSTOM, "自定义"),
+    )
+    org_id = models.IntegerField(verbose_name="所属组织", is_default_query=False, is_base_query=False)
+
+    CONNECT_STATUS_CHECKING = "checking"
+    CONNECT_STATUS_SUCCESS = "success"
+    CONNECT_STATUS_FAIL = "fail"
+    CONNECT_STATUS_CHOICES = (
+        (CONNECT_STATUS_CHECKING, "检查中", "blue"),
+        (CONNECT_STATUS_SUCCESS, "连接成功", "green"),
+        (CONNECT_STATUS_FAIL, "连接失败", "red"),
+    )
+
+    # ========== 新增：API 认证方式枚举 ==========
+    AUTH_TYPE_NONE = "none"
+    AUTH_TYPE_LOGIN_URL = "login_url"
+    API_AUTH_TYPE_CHOICES = (
+        (AUTH_TYPE_NONE, "不认证"),
+        (AUTH_TYPE_LOGIN_URL, "登录URL认证"),
+    )
+    # ============================================
+
+    name = models.CharField(max_length=50, verbose_name="数据源名称")
+    source_type = models.CharField(max_length=50, verbose_name="数据源类型", choices=SOURCE_TYPE_CHOICES)
+    db_type = models.CharField(max_length=50, verbose_name="数据库类型", choices=DB_TYPE_CHOICES, null=True, blank=True)
+    db_host = models.CharField(max_length=50, verbose_name="数据库主机", null=True, blank=True)
+    db_port = models.IntegerField(verbose_name="数据库端口", null=True, blank=True)
+    db_user = models.CharField(max_length=50, verbose_name="数据库用户", null=True, blank=True)
+    db_password = models.CharField(max_length=50, verbose_name="数据库密码", null=True, blank=True)
+    db_name = models.CharField(max_length=50, verbose_name="数据库名称", null=True, blank=True)
+    db_options = models.JSONField(verbose_name="数据库选项", is_default_query=False, is_base_query=False, default=dict)
+    db_connect_status = models.CharField(
+        max_length=50,
+        verbose_name="数据库连接状态",
+        choices=CONNECT_STATUS_CHOICES,
+        editable=False,
+        null=True,
+        blank=True,
+    )
+    db_connect_message = models.TextField(verbose_name="数据库连接消息", null=True, blank=True, editable=False)
+
+    custom_func = models.CharField(max_length=50, verbose_name="自定义采集函数名称", null=True, blank=True)
+    custom_func_kwargs = models.JSONField(
+        verbose_name="自定义采集函数参数", is_default_query=False, is_base_query=False, default=dict
+    )
+
+    # ========== 调用并发与频率控制（作用于相同数据源下的所有任务） ==========
+    # 相同数据源下，所有任务是否能够并发执行
+    task_concurrency_enabled = models.BooleanField(
+        default=False,
+        verbose_name="任务可并发",
+        help_text="相同数据源下，所有任务是否能够并发执行。"
+        "开启后受【最大任务并发数】限制；关闭后任务串行执行，调用间隔必须达到【任务调用最小间隔】。",
+        is_default_query=False,
+        is_base_query=False,
+    )
+    # 并发时允许同时运行的最大任务数（仅当 task_concurrency_enabled=True 时生效）
+    task_max_concurrency = models.IntegerField(
+        default=1,
+        verbose_name="最大任务并发数",
+        help_text="相同数据源下允许同时运行的最大任务数（仅在【任务可并发】开启时生效），至少为 1。",
+        is_default_query=False,
+        is_base_query=False,
+    )
+    # 不能并发时，两次调用之间必须间隔的最小秒数（仅当 task_concurrency_enabled=False 时生效）
+    task_call_interval = models.IntegerField(
+        default=0,
+        verbose_name="任务调用最小间隔(秒)",
+        help_text="相同数据源下任务不可并发时，两次调用之间必须间隔的最小秒数（仅在【任务可并发】关闭时生效）。",
+        is_default_query=False,
+        is_base_query=False,
+    )
+    # =====================================================================
+
+    class Meta:
+        verbose_name = "数据源"
+        pageinfo = True
+        hide_field_list = {
+            "org_id": ("组织ID", "r"),
+        }
+        # 敏感字段隐藏
+        hide_json = ["db_password", "oauth2_client_secret"]
+        list_json = {
+            "org_id": ("组织ID", "r"),
+        }
+
+
+EXPORT_RECORD_STATUS_CHECKING = "checking"
+EXPORT_RECORD_STATUS_WAIT = "wait"
+EXPORT_RECORD_STATUS_RUNNING = "running"
+EXPORT_RECORD_STATUS_SUCCESS = "success"
+EXPORT_RECORD_STATUS_FAIL = "fail"
+EXPORT_RECORD_STATUS_STOPPING = "stopping"
+EXPORT_RECORD_STATUS_STOPPED = "stopped"
+
+EXPORT_RECORD_STATUS_CHOICES = (
+    (EXPORT_RECORD_STATUS_CHECKING, "检查中", "blue"),
+    (EXPORT_RECORD_STATUS_RUNNING, "运行中", "blue"),
+    (EXPORT_RECORD_STATUS_SUCCESS, "成功", "green"),
+    (EXPORT_RECORD_STATUS_FAIL, "失败", "red"),
+    (EXPORT_RECORD_STATUS_STOPPING, "停止中", "yellow"),
+    (EXPORT_RECORD_STATUS_STOPPED, "已停止", "orange"),
+    (EXPORT_RECORD_STATUS_WAIT, "等待中", "yellow"),
+)
+EXPLAIN_STATUS_DEFAULT = "default"
+EXPLAIN_STATUS_CHECKING = "checking"
+EXPLAIN_STATUS_A1 = "A1"
+EXPLAIN_STATUS_A2 = "A2"
+EXPLAIN_STATUS_B1 = "B1"
+EXPLAIN_STATUS_B2 = "B2"
+EXPLAIN_STATUS_C1 = "C1"
+EXPLAIN_STATUS_C2 = "C2"
+EXPLAIN_STATUS_D1 = "D1"
+EXPLAIN_STATUS_D2 = "D2"
+EXPLAIN_STATUS_E1 = "E1"
+EXPLAIN_STATUS_FAILED = "failed"
+EXPLAIN_STATUS_CHOICES = (
+    (EXPLAIN_STATUS_DEFAULT, "未检测", "default"),
+    (EXPLAIN_STATUS_CHECKING, "检测中", "default"),
+    (EXPLAIN_STATUS_A1, "最佳(A1)", "green"),
+    (EXPLAIN_STATUS_A2, "优秀(A2)", "green"),
+    (EXPLAIN_STATUS_B1, "良好(B1)", "blue"),
+    (EXPLAIN_STATUS_B2, "良好(B2)", "blue"),
+    (EXPLAIN_STATUS_C1, "一般(C1)", "yellow"),
+    (EXPLAIN_STATUS_C2, "一般(C2)", "yellow"),
+    (EXPLAIN_STATUS_D1, "较差(D1)", "red"),
+    (EXPLAIN_STATUS_D2, "较差(D2)", "red"),
+    (EXPLAIN_STATUS_E1, "极差(E1)", "red"),
+    (EXPLAIN_STATUS_FAILED, "失败", "red"),
+)
+
+
+class SyncExportTask(BaseModel):
+    org_id = models.IntegerField(verbose_name="所属组织", is_default_query=False, is_base_query=False)
+    name = models.CharField(max_length=50, verbose_name="任务名称")
+    data_source = models.ForeignKey(SyncDataSource, on_delete=CASCADE, verbose_name="数据源")
+    model_type = models.ForeignKey(SyncModelType, on_delete=CASCADE, verbose_name="数据模型")
+    sql = models.TextField(verbose_name="SQL", null=True, blank=True)
+    explain_status = models.CharField(
+        max_length=50,
+        verbose_name="SQL性能",
+        editable=False,
+        choices=EXPLAIN_STATUS_CHOICES,
+        default=EXPLAIN_STATUS_DEFAULT,
+    )
+    explain_json = models.JSONField(verbose_name="SQL性能结果", editable=False, null=True, blank=True)
+    last_export_time = models.DateTimeField(verbose_name="最后导出时间", editable=False, null=True, blank=True)
+    last_export_unique_hash = models.CharField(
+        max_length=32, verbose_name="最后导出唯一标识", editable=False, null=True, blank=True
+    )
+    last_status = models.CharField(
+        max_length=50,
+        verbose_name="最后状态",
+        editable=False,
+        null=True,
+        blank=True,
+        choices=EXPORT_RECORD_STATUS_CHOICES,
+    )
+    last_message = models.TextField(verbose_name="最后消息", editable=False, null=True, blank=True)
+    # custom fields
+    custom_func_kwargs = models.JSONField(
+        verbose_name="自定义采集函数参数",
+        is_default_query=False,
+        is_base_query=False,
+        help_text="此参数会合并数据源的自定义采集函数参数, 此任务参数的优先级较高",
+        default=dict
+    )
+    process_list = models.JSONField(
+        verbose_name="处理函数列表", is_default_query=False, is_base_query=False, default=list
+    )
+
+    # ========== 调用并发与频率控制（作用于相同的任务自身） ==========
+    # 相同任务自身是否能够并发执行
+    self_concurrency_enabled = models.BooleanField(
+        default=False,
+        verbose_name="自身可并发",
+        help_text="相同任务是否能够并发执行。"
+        "开启后受【自身最大并发数】限制；关闭后同一任务串行执行，调用间隔必须达到【自身调用最小间隔】。",
+        is_default_query=False,
+        is_base_query=False,
+    )
+    # 并发时允许同时运行的最大实例数（仅当 self_concurrency_enabled=True 时生效）
+    self_max_concurrency = models.IntegerField(
+        default=1,
+        verbose_name="自身最大并发数",
+        help_text="相同任务允许同时运行的最大实例数（仅在【自身可并发】开启时生效），至少为 1。",
+        is_default_query=False,
+        is_base_query=False,
+    )
+    # 不能并发时，两次调用之间必须间隔的最小秒数（仅当 self_concurrency_enabled=False 时生效）
+    self_call_interval = models.IntegerField(
+        default=0,
+        verbose_name="自身调用最小间隔(秒)",
+        help_text="相同任务不可并发时，两次调用之间必须间隔的最小秒数（仅在【自身可并发】关闭时生效）。",
+        is_default_query=False,
+        is_base_query=False,
+    )
+    # ===============================================================
+    data_start_date = models.DateField(verbose_name="数据开始日期", null=True, blank=True)
+
+    class Meta:
+        verbose_name = "数据导出任务"
+        pageinfo = True
+        hide_field_list = {
+            "id": ["ID", "r,int"],
+            "data_source_id": ["数据源ID", "int"],
+            "model_type_id": ["数据类型ID", "int"],
+            "explain_json": ["SQL性能结果", "json"],
+        }
+        list_json = {
+            "data_source__name": ("数据源", "r"),
+            "model_type__name": ("数据类型", "r"),
+        }
+
+
+class SyncExportTaskHistory(BaseModel):
+    export_task = models.ForeignKey(SyncExportTask, on_delete=CASCADE, verbose_name="任务", null=True, blank=True)
+    sql = models.TextField(verbose_name="SQL")
+
+    class Meta:
+        verbose_name = "数据导出任务历史"
+        ordering = ["id"]
+        pageinfo = True
+        hide_field_list = {
+            "id": ["ID", "r,int"],
+            "export_task_id": ["任务ID", "int"],
+        }
+        list_json = {
+            "export_task__name": ("任务", "r"),
+        }
+
+
+class SyncDataFingerprint(BaseModel):
+    org_id = models.IntegerField(verbose_name="所属组织", is_default_query=False, is_base_query=False)
+    export_task = models.ForeignKey(SyncExportTask, on_delete=CASCADE, verbose_name="任务")
+    origin_recode_unique_hash = models.CharField(max_length=32, verbose_name="原始数据唯一标识", db_index=True)
+    fingerprint = models.CharField(max_length=32, verbose_name="数据指纹")
+
+    class Meta:
+        verbose_name = "数据指纹"
+        unique_together = ("export_task", "origin_recode_unique_hash")
+
+
+class SyncExportRecord(BaseModel):
+    CHECK_RECORD_STATUS_COMPLETE = "complete"
+    CHECK_RECORD_STATUS_WAIT = "wait"
+    CHECK_RECORD_STATUS_CHECKING = "checking"
+
+    CHECK_RECORD_STATUS_CHOICES = (
+        (CHECK_RECORD_STATUS_COMPLETE, "完成", "green"),
+        (CHECK_RECORD_STATUS_WAIT, "等待检查", "blue"),
+        (CHECK_RECORD_STATUS_CHECKING, "检查中", "yellow"),
+    )
+    org_id = models.IntegerField(verbose_name="所属组织", is_default_query=False, is_base_query=False)
+    export_task = models.ForeignKey(SyncExportTask, on_delete=CASCADE, verbose_name="任务")
+    original_sql = models.TextField(verbose_name="原始 SQL")
+    exec_sql = models.TextField(verbose_name="执行 SQL")
+    export_status = models.CharField(max_length=50, verbose_name="导出状态", choices=EXPORT_RECORD_STATUS_CHOICES)
+    check_status = models.CharField(
+        max_length=50, verbose_name="检查状态", choices=CHECK_RECORD_STATUS_CHOICES, null=True, blank=True
+    )
+    message = models.TextField(verbose_name="导出消息")
+    # 本次同步对应的数据日期（按天同步，前闭后开区间的起始日）
+    data_date = models.DateField(verbose_name="数据日期", null=True, blank=True)
+    # 本次导出数据的唯一标识（基于 is_unique 字段计算的 md5）列表
+    export_unique_hash_list = models.JSONField(verbose_name="导出数据唯一标识列表", null=True, blank=True)
+    export_count = models.IntegerField(verbose_name="导出数据数量", default=0)
+    export_total_count = models.IntegerField(
+        verbose_name="导出数据总数", default=0, help_text="导出数据总数,包含补充数据量"
+    )
+    last_check_message = models.TextField(verbose_name="最后补录消息", editable=False, null=True, blank=True)
+    last_check_record_id = models.IntegerField(verbose_name="最后补录记录ID", editable=False, null=True, blank=True)
+    success_check_record_count = models.IntegerField(verbose_name="成功补录数量", default=0)
+    last_check_time = models.DateTimeField(verbose_name="最后补录时间", editable=False, null=True, blank=True)
+
+    class Meta:
+        verbose_name = "数据导出记录"
+        ordering = ["id"]
+        pageinfo = True
+        is_operation = False
+        hide_json = ["export_unique_hash_list"]
+        hide_field_list = {
+            "id": ["ID", "r,int"],
+            "export_task_id": ["任务ID", "int"],
+        }
+        list_json = {
+            "export_task__name": ("任务", "r"),
+        }
+        select_label_fields = "id"
+
+
+class SyncExportCheckRecord(BaseModel):
+    org_id = models.IntegerField(verbose_name="所属组织", is_default_query=False, is_base_query=False)
+    export_task = models.ForeignKey(SyncExportTask, on_delete=CASCADE, verbose_name="任务")
+    export_record = models.ForeignKey(SyncExportRecord, on_delete=CASCADE, verbose_name="导出记录")
+    original_sql = models.TextField(verbose_name="原始 SQL")
+    exec_sql = models.TextField(verbose_name="执行 SQL")
+    export_status = models.CharField(max_length=50, verbose_name="导出状态", choices=EXPORT_RECORD_STATUS_CHOICES)
+    message = models.TextField(verbose_name="导出消息")
+    export_unique_hash_list = models.JSONField(verbose_name="导出数据唯一标识列表", null=True, blank=True)
+    export_count = models.IntegerField(verbose_name="导出数据数量", default=0)
+
+    class Meta:
+        verbose_name = "数据导出补充记录"
+        ordering = ["id"]
+        pageinfo = True
+        is_operation = False
+        hide_json = ["export_unique_hash_list"]
+        hide_field_list = {
+            "id": ["ID", "r,int"],
+            "export_task_id": ["任务ID", "int"],
+            "export_record_id": ["导出记录ID", "int"],
+        }
+        list_json = {
+            "export_task__name": ("任务", "r"),
+        }

lysync/signals/__init__.py

@@ -0,0 +1,3 @@
+# 导入子模块以注册 Django 信号（post_save 等）。
+# 批量更新信号在 apps.ready() 中通过 connect_batch_update_signal() 注册。
+from . import datasource  # noqa: F401

lysync/signals/datasource.py

@@ -0,0 +1,46 @@
+"""数据源（SyncDataSource）信号。
+
+移植自 costexcel/signals/datasource.py，适配当前模型：
+
+* ``DataSource`` -> :class:`~lysync.models.SyncDataSource`
+* 连接检查任务 ``check_datasource_connect`` 来自 ``lysync.tasks``，队列 ``lysync_export_queue``
+* 批量更新信号来自宿主 ``lybase.signals.batch_update``；该模块为宿主依赖，
+  缺失时静默跳过，不影响本 app 的其余功能。
+
+行为：
+* ``post_save``：每次保存数据源都会异步触发一次连接检查
+  （任务内部以 ``db_connect_status == CHECKING`` 作为开关，避免信号回环）。
+* 批量更新：对同一批中的**每个**数据源各触发一次检查。
+  原 costexcel 按 ``(excel_type_id, org_id)`` 去重，但 ``SyncDataSource``
+  已无 ``excel_type_id`` 且无复用关系，这里改为按 ``id`` 去重，
+  保证每个数据源都被检查且不会在同一批内重复检查。
+"""
+
+from django.db.models.signals import pre_save, post_save
+from django.dispatch import receiver
+from loguru import logger
+from lysync.utils import get_queue_name
+from util.middleware import get_request_id
+
+from lysync.models import SyncDataSource
+
+logger = logger.patch(lambda record: record["extra"].update(**get_request_id()))
+
+
+@receiver(post_save, sender=SyncDataSource)
+def datasource_post_save(sender, instance, **kwargs):
+    from lysync.tasks import check_datasource_connect
+
+    logger.info("run check_datasource_connect")
+    if instance.is_active and instance.source_type == SyncDataSource.SOURCE_TYPE_DB:
+        check_datasource_connect.apply_async(
+            kwargs={"datasource_id": instance.id},
+            queue=get_queue_name(),
+        )
+
+
+@receiver(pre_save, sender=SyncDataSource)
+def datasource_pre_save(sender, instance, **kwargs):
+    if not instance.id and instance.source_type == SyncDataSource.SOURCE_TYPE_DB:
+        instance.db_connect_status = SyncDataSource.CONNECT_STATUS_CHECKING
+        instance.db_connect_message = ""