lysync 1.0.0__py3-none-any.whl

lysync/funcs/process.py

@@ -0,0 +1,286 @@
+"""内置数据处理函数。
+
+这些函数通过 :func:`lysync.funcs.registry.register_func` 注册，可在
+``process_list`` 中按 ``func`` 名称引用。
+
+* 作用于单个字段值的函数（``scope=FIELD``）签名：``func(value, **kwargs) -> value``
+* 作用于整批数据的函数（``scope=DATASET``）签名：``func(rows, **kwargs) -> rows``
+"""
+
+from __future__ import annotations
+
+import statistics
+
+from .registry import FuncType, ProcessScope, register_func
+
+# ============================ 字段级处理函数 (FIELD) ============================
+
+
+@register_func("strip", FuncType.PROCESS, scope=ProcessScope.FIELD)
+def strip(value, **kwargs):
+    """去除字符串首尾空白；非字符串原样返回。"""
+    if isinstance(value, str):
+        return value.strip()
+    return value
+
+
+@register_func("to_int", FuncType.PROCESS, scope=ProcessScope.FIELD)
+def to_int(value, default=None, **kwargs):
+    """转为 int；失败返回 ``default``。"""
+    if value is None or value == "":
+        return default
+    try:
+        if isinstance(value, str):
+            value = value.strip().replace(",", "")
+        return int(float(value))
+    except (ValueError, TypeError):
+        return default
+
+
+@register_func("to_float", FuncType.PROCESS, scope=ProcessScope.FIELD)
+def to_float(value, default=None, **kwargs):
+    """转为 float；失败返回 ``default``。"""
+    if value is None or value == "":
+        return default
+    try:
+        if isinstance(value, str):
+            value = value.strip().replace(",", "")
+        return float(value)
+    except (ValueError, TypeError):
+        return default
+
+
+@register_func("default_if_none", FuncType.PROCESS, scope=ProcessScope.FIELD)
+def default_if_none(value, default=None, **kwargs):
+    """当值为 ``None`` / 空字符串时返回 ``default``。"""
+    if value is None:
+        return default
+    if isinstance(value, str) and value.strip() == "":
+        return default
+    return value
+
+
+@register_func("upper", FuncType.PROCESS, scope=ProcessScope.FIELD)
+def upper(value, **kwargs):
+    """字符串转大写。"""
+    return value.upper() if isinstance(value, str) else value
+
+
+@register_func("lower", FuncType.PROCESS, scope=ProcessScope.FIELD)
+def lower(value, **kwargs):
+    """字符串转小写。"""
+    return value.lower() if isinstance(value, str) else value
+
+
+@register_func("replace", FuncType.PROCESS, scope=ProcessScope.FIELD)
+def replace(value, old="", new="", **kwargs):
+    """字符串替换，``old`` -> ``new``。"""
+    if not isinstance(value, str):
+        return value
+    return value.replace(old, new)
+
+
+@register_func("bool_from_str", FuncType.PROCESS, scope=ProcessScope.FIELD)
+def bool_from_str(value, true_values=None, false_values=None, default=None, **kwargs):
+    """依据字符串内容转为布尔值。
+
+    ``true_values`` / ``false_values`` 均可为逗号分隔字符串或列表，默认
+    真值：``是,yes,y,true,1``；假值：``否,no,n,false,0``。
+    """
+    if value is None:
+        return default
+    if isinstance(value, bool):
+        return value
+
+    def _norm(x):
+        return (x or "").strip().lower()
+
+    if isinstance(true_values, str):
+        true_values = [t.strip() for t in true_values.split(",")]
+    if isinstance(false_values, str):
+        false_values = [t.strip() for t in false_values.split(",")]
+    true_values = [t.lower() for t in (true_values or ["是", "yes", "y", "true", "1"])]
+    false_values = [f.lower() for f in (false_values or ["否", "no", "n", "false", "0"])]
+
+    v = _norm(value)
+    if v in true_values:
+        return True
+    if v in false_values:
+        return False
+    return default
+
+
+# ============================ 数据集级处理函数 (DATASET) ============================
+
+
+@register_func("filter_by_field", FuncType.PROCESS, scope=ProcessScope.DATASET)
+def filter_by_field(rows, field=None, op="==", value=None, **kwargs):
+    """过滤整批数据，仅保留满足条件的行。
+
+    ``op`` 支持 ``==, !=, >, >=, <, <=, in, not_in, contains``。
+    """
+    if not field:
+        return rows
+    result = []
+    for row in rows:
+        v = row.get(field)
+        keep = False
+        if op == "==":
+            keep = v == value
+        elif op == "!=":
+            keep = v != value
+        elif op == ">":
+            keep = v is not None and v > value
+        elif op == ">=":
+            keep = v is not None and v >= value
+        elif op == "<":
+            keep = v is not None and v < value
+        elif op == "<=":
+            keep = v is not None and v <= value
+        elif op == "in":
+            keep = v in (value or [])
+        elif op == "not_in":
+            keep = v not in (value or [])
+        elif op == "contains":
+            keep = value is not None and v is not None and str(value) in str(v)
+        if keep:
+            result.append(row)
+    return result
+
+
+@register_func("deduplicate", FuncType.PROCESS, scope=ProcessScope.DATASET)
+def deduplicate(rows, keys=None, keep="first", **kwargs):
+    """按 ``keys``（字段名列表，缺省使用全部字段）去重。
+
+    ``keep`` 为 ``first`` / ``last``。
+    """
+    if not rows:
+        return rows
+    if isinstance(keys, str):
+        keys = [k.strip() for k in keys.split(",")]
+    seen = {}
+    out = []
+    for idx, row in enumerate(rows):
+        if keys:
+            key = tuple(row.get(k) for k in keys)
+        else:
+            key = tuple(sorted(row.items()))
+        if key in seen:
+            if keep == "last":
+                # 用当前行替换之前保留的行
+                out[seen[key]] = row
+            continue
+        seen[key] = len(out)
+        out.append(row)
+    return out
+
+
+@register_func("sort_rows", FuncType.PROCESS, scope=ProcessScope.DATASET)
+def sort_rows(rows, field=None, desc=False, **kwargs):
+    """按 ``field`` 对整批数据排序（原地返回新列表）。"""
+    if not field or not rows:
+        return rows
+
+    def _key(r):
+        v = r.get(field)
+        return (v is None, v)
+
+    return sorted(rows, key=_key, reverse=bool(desc))
+
+
+@register_func("query_counter_diff_robust", FuncType.PROCESS, scope=ProcessScope.DATASET)
+def query_counter_diff_robust(
+    rows,
+    counter_field="value",
+    time_field="time",
+    lookback="auto",
+    recover_ratio=0.8,
+    recover_window=30,
+    min_valid_ratio=0.5,
+    spike_recover_abs=100,
+    reset_abs_threshold=100,
+    spike_abs_threshold=100,
+    diff_field="diff",
+    **kwargs,
+):
+    """对计数器型指标做“增量(diff)”计算，并对计数器复位 / 尖峰做鲁棒处理。
+
+    适用场景：水/电/气等只增不减的累计读数，需要转换为每段增量。
+
+    Args:
+        counter_field: 计数字段名。
+        time_field:    时间字段名（用于排序，支持 datetime / 数值 / 可解析字符串）。
+        reset_abs_threshold: 当 ``当前值 - 上一条值 <= -该阈值`` 时判定为计数器复位，diff 归零。
+        spike_abs_threshold: 当增量绝对值超过该阈值时判定为疑似尖峰。
+        spike_recover_abs:   用于恢复尖峰的“正常增量”绝对值上限，超过则无法恢复、保留原值。
+        recover_window:      恢复尖峰时参考的近邻正常增量窗口大小（条数）。
+        lookback / recover_ratio / min_valid_ratio: 预留参数（保持与配置示例兼容，
+            当前版本在样本不足时退化为保留原增量）。
+
+    Returns:
+        在原 ``rows`` 的每一行追加 ``diff_field``（默认 ``diff``）字段后返回。
+    """
+    if not rows:
+        return rows
+
+    def _ts(r):
+        v = r.get(time_field)
+        if v is None:
+            return 0
+        if hasattr(v, "timestamp"):
+            return v.timestamp()
+        try:
+            return float(v)
+        except (TypeError, ValueError):
+            return 0
+
+    sorted_rows = sorted(rows, key=_ts)
+
+    valid_diffs = []
+    prev_value = None
+    for r in sorted_rows:
+        value = r.get(counter_field)
+        try:
+            value = float(value) if value is not None else None
+        except (TypeError, ValueError):
+            value = None
+
+        if value is None:
+            r[diff_field] = None
+            continue
+
+        if prev_value is None:
+            # 第一条有效样本，无法计算增量
+            r[diff_field] = 0.0
+            prev_value = value
+            continue
+
+        raw_diff = value - prev_value
+
+        # 计数器复位：累计值大幅下降
+        if raw_diff <= -reset_abs_threshold:
+            r[diff_field] = 0.0
+            prev_value = value
+            continue
+
+        # 疑似尖峰：增量异常偏大，尝试用近邻正常增量恢复
+        if raw_diff >= spike_abs_threshold:
+            if valid_diffs:
+                window = valid_diffs[-recover_window:] if recover_window else valid_diffs
+                median_diff = statistics.median(window)
+                if abs(median_diff) <= spike_recover_abs:
+                    r[diff_field] = median_diff
+                    valid_diffs.append(median_diff)
+                    prev_value = value
+                    continue
+            # 无法恢复，保留原始增量但仍计入统计
+            r[diff_field] = raw_diff
+            valid_diffs.append(raw_diff)
+            prev_value = value
+            continue
+
+        r[diff_field] = raw_diff
+        valid_diffs.append(raw_diff)
+        prev_value = value
+
+    return sorted_rows

lysync/funcs/registry.py

@@ -0,0 +1,116 @@
+"""自定义函数注册中心。
+
+通过 :func:`register_func` 装饰器把处理函数登记到注册表中，供两处引用：
+
+1. 数据处理链 ``process_list`` —— 在 :class:`~lysync.models.SyncModelField`、
+   :class:`~lysync.models.SyncModelType`、:class:`~lysync.models.SyncExportTask`
+   上配置，作为数据加工步骤。
+2. 采集自定义函数 ``custom_func`` —— 在 :class:`~lysync.models.SyncDataSource`
+   上配置，作为 ``source_type='custom'`` 时的数据源采集逻辑。
+
+装饰器需要两个必填参数：
+
+* ``config_name`` : 配置名称，即在 ``process_list`` / ``custom_func`` 字段中
+  填写的名字。同一个名字可以同时存在于 “数据处理” 与 “采集” 两个命名空间。
+* ``func_type``   : 函数类型，取值 ``FuncType.PROCESS`` 或 ``FuncType.COLLECT``。
+
+对于 ``func_type='process'`` 的函数，还可用 ``scope`` 声明作用范围：
+
+* ``ProcessScope.FIELD``   : 作用于**单个字段的值**，签名 ``func(value, **kwargs) -> value``。
+* ``ProcessScope.DATASET`` : 作用于**整批数据**，签名 ``func(rows, **kwargs) -> rows``。
+
+采集函数（``func_type='collect'``）签名为 ``func(**kwargs) -> list[dict]``，
+返回的列表中每个字典的键应与目标模型的字段名（``SyncModelField.field_name``）一致。
+"""
+
+from __future__ import annotations
+
+import logging
+
+logger = logging.getLogger(__name__)
+
+
+class FuncType:
+    PROCESS = "process"
+    COLLECT = "collect"
+
+
+class ProcessScope:
+    FIELD = "field"
+    DATASET = "dataset"
+
+
+# config_name -> {"func": callable, "scope": str}
+PROCESS_FUNCS: dict = {}
+
+# config_name -> callable
+COLLECT_FUNCS: dict = {}
+
+
+def register_func(config_name: str, func_type: str, scope: str = ProcessScope.DATASET, override: bool = False):
+    """注册自定义函数的装饰器。
+
+    Args:
+        config_name: 配置名称（在 ``process_list`` / ``custom_func`` 中引用的名字）。
+        func_type: 函数类型，``FuncType.PROCESS`` 或 ``FuncType.COLLECT``。
+        scope: 仅对 ``func_type='process'`` 有效，``ProcessScope.FIELD`` 或
+            ``ProcessScope.DATASET``，默认 ``DATASET``。
+        override: 是否覆盖同名已注册函数，默认 ``False``（重复注册仅告警）。
+    """
+    if func_type not in (FuncType.PROCESS, FuncType.COLLECT):
+        raise ValueError(f"func_type 仅支持 {FuncType.PROCESS}/{FuncType.COLLECT}，实际为 {func_type!r}")
+    if func_type == FuncType.PROCESS and scope not in (ProcessScope.FIELD, ProcessScope.DATASET):
+        raise ValueError(f"process 函数 scope 仅支持 {ProcessScope.FIELD}/{ProcessScope.DATASET}，实际为 {scope!r}")
+
+    def decorator(func):
+        if func_type == FuncType.PROCESS:
+            if config_name in PROCESS_FUNCS and not override:
+                logger.warning("重复注册 process 函数: %s（使用 override=True 可覆盖）", config_name)
+                return func
+            PROCESS_FUNCS[config_name] = {"func": func, "scope": scope}
+        else:
+            if config_name in COLLECT_FUNCS and not override:
+                logger.warning("重复注册 collect 函数: %s（使用 override=True 可覆盖）", config_name)
+                return func
+            COLLECT_FUNCS[config_name] = func
+
+        func._lysync_config_name = config_name
+        func._lysync_func_type = func_type
+        func._lysync_scope = scope if func_type == FuncType.PROCESS else None
+        return func
+
+    return decorator
+
+
+def register_process(config_name: str, scope: str = ProcessScope.DATASET, override: bool = False):
+    """便捷装饰器：注册一个数据处理函数。"""
+    return register_func(config_name, FuncType.PROCESS, scope=scope, override=override)
+
+
+def register_collect(config_name: str, override: bool = False):
+    """便捷装饰器：注册一个采集函数。"""
+    return register_func(config_name, FuncType.COLLECT, override=override)
+
+
+def get_process_func(config_name: str):
+    """返回 ``(func, scope)``，找不到时抛出 ``KeyError``。"""
+    item = PROCESS_FUNCS.get(config_name)
+    if not item:
+        raise KeyError(f"未找到 process 函数: {config_name!r}，请确认已通过 @register_func 注册")
+    return item["func"], item["scope"]
+
+
+def get_collect_func(config_name: str):
+    """返回采集函数，找不到时抛出 ``KeyError``。"""
+    func = COLLECT_FUNCS.get(config_name)
+    if not func:
+        raise KeyError(f"未找到 collect 函数: {config_name!r}，请确认已通过 @register_func 注册")
+    return func
+
+
+def list_registered():
+    """返回当前已注册函数的概览（用于调试/接口展示）。"""
+    return {
+        "process": {name: meta["scope"] for name, meta in PROCESS_FUNCS.items()},
+        "collect": list(COLLECT_FUNCS.keys()),
+    }

lysync/migrations/0001_initial.py

@@ -0,0 +1,207 @@
+# Generated by Django 3.2.25 on 2026-07-11 11:34
+
+from django.db import migrations, models
+import django.db.models.deletion
+import django.utils.timezone
+import util.base_model_fields
+import util.basemodel
+
+
+class Migration(migrations.Migration):
+
+    initial = True
+
+    dependencies = [
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='SyncDataSource',
+            fields=[
+                ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('is_active', util.base_model_fields.BooleanField(choices=[(True, '可用'), (False, '禁用')], default=True, verbose_name='是否可用')),
+                ('create_time', util.base_model_fields.DateTimeField(db_index=True, default=django.utils.timezone.now, verbose_name='创建时间')),
+                ('update_time', util.base_model_fields.DateTimeField(auto_now=True, db_index=True, verbose_name='更新时间')),
+                ('org_id', util.base_model_fields.IntegerField(verbose_name='所属组织')),
+                ('name', util.base_model_fields.CharField(max_length=50, verbose_name='数据源名称')),
+                ('source_type', util.base_model_fields.CharField(choices=[('db', '数据库'), ('custom', '自定义')], max_length=50, verbose_name='数据源类型')),
+                ('db_type', util.base_model_fields.CharField(blank=True, choices=[('mysql', 'MySQL'), ('tsql', 'SQL Server')], max_length=50, null=True, verbose_name='数据库类型')),
+                ('db_host', util.base_model_fields.CharField(blank=True, max_length=50, null=True, verbose_name='数据库主机')),
+                ('db_port', util.base_model_fields.IntegerField(blank=True, null=True, verbose_name='数据库端口')),
+                ('db_user', util.base_model_fields.CharField(blank=True, max_length=50, null=True, verbose_name='数据库用户')),
+                ('db_password', util.base_model_fields.CharField(blank=True, max_length=50, null=True, verbose_name='数据库密码')),
+                ('db_name', util.base_model_fields.CharField(blank=True, max_length=50, null=True, verbose_name='数据库名称')),
+                ('db_options', util.base_model_fields.JSONField(default=dict, verbose_name='数据库选项')),
+                ('db_connect_status', util.base_model_fields.CharField(blank=True, choices=[('checking', '检查中'), ('success', '连接成功'), ('fail', '连接失败')], editable=False, max_length=50, null=True, verbose_name='数据库连接状态')),
+                ('db_connect_message', util.base_model_fields.TextField(blank=True, editable=False, null=True, verbose_name='数据库连接消息')),
+                ('custom_func', util.base_model_fields.CharField(blank=True, max_length=50, null=True, verbose_name='自定义采集函数名称')),
+                ('custom_func_kwargs', util.base_model_fields.JSONField(default=dict, verbose_name='自定义采集函数参数')),
+                ('task_concurrency_enabled', util.base_model_fields.BooleanField(default=False, help_text='相同数据源下，所有任务是否能够并发执行。开启后受【最大任务并发数】限制；关闭后任务串行执行，调用间隔必须达到【任务调用最小间隔】。', verbose_name='任务可并发')),
+                ('task_max_concurrency', util.base_model_fields.IntegerField(default=1, help_text='相同数据源下允许同时运行的最大任务数（仅在【任务可并发】开启时生效），至少为 1。', verbose_name='最大任务并发数')),
+                ('task_call_interval', util.base_model_fields.IntegerField(default=0, help_text='相同数据源下任务不可并发时，两次调用之间必须间隔的最小秒数（仅在【任务可并发】关闭时生效）。', verbose_name='任务调用最小间隔(秒)')),
+            ],
+            options={
+                'verbose_name': '数据源',
+            },
+            bases=(models.Model, util.basemodel.JSONBaseMixin, util.basemodel.ModefyMixin),
+        ),
+        migrations.CreateModel(
+            name='SyncExportTask',
+            fields=[
+                ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('is_active', util.base_model_fields.BooleanField(choices=[(True, '可用'), (False, '禁用')], default=True, verbose_name='是否可用')),
+                ('create_time', util.base_model_fields.DateTimeField(db_index=True, default=django.utils.timezone.now, verbose_name='创建时间')),
+                ('update_time', util.base_model_fields.DateTimeField(auto_now=True, db_index=True, verbose_name='更新时间')),
+                ('org_id', util.base_model_fields.IntegerField(verbose_name='所属组织')),
+                ('name', util.base_model_fields.CharField(max_length=50, verbose_name='任务名称')),
+                ('sql', util.base_model_fields.TextField(blank=True, null=True, verbose_name='SQL')),
+                ('explain_status', util.base_model_fields.CharField(choices=[('default', '未检测'), ('checking', '检测中'), ('A1', '最佳(A1)'), ('A2', '优秀(A2)'), ('B1', '良好(B1)'), ('B2', '良好(B2)'), ('C1', '一般(C1)'), ('C2', '一般(C2)'), ('D1', '较差(D1)'), ('D2', '较差(D2)'), ('E1', '极差(E1)'), ('failed', '失败')], default='default', editable=False, max_length=50, verbose_name='SQL性能')),
+                ('explain_json', util.base_model_fields.JSONField(blank=True, null=True, verbose_name='SQL性能结果')),
+                ('last_export_time', util.base_model_fields.DateTimeField(blank=True, editable=False, null=True, verbose_name='最后导出时间')),
+                ('last_export_unique_hash', util.base_model_fields.CharField(blank=True, editable=False, max_length=32, null=True, verbose_name='最后导出唯一标识')),
+                ('last_status', util.base_model_fields.CharField(blank=True, choices=[('checking', '检查中'), ('running', '运行中'), ('success', '成功'), ('fail', '失败'), ('stopping', '停止中'), ('stopped', '已停止'), ('wait', '等待中')], editable=False, max_length=50, null=True, verbose_name='最后状态')),
+                ('last_message', util.base_model_fields.TextField(blank=True, editable=False, null=True, verbose_name='最后消息')),
+                ('custom_func_kwargs', util.base_model_fields.JSONField(help_text='此参数会合并数据源的自定义采集函数参数, 此任务参数的优先级较高', verbose_name='自定义采集函数参数')),
+                ('process_list', util.base_model_fields.JSONField(default=list, verbose_name='处理函数列表')),
+                ('self_concurrency_enabled', util.base_model_fields.BooleanField(default=False, help_text='相同任务是否能够并发执行。开启后受【自身最大并发数】限制；关闭后同一任务串行执行，调用间隔必须达到【自身调用最小间隔】。', verbose_name='自身可并发')),
+                ('self_max_concurrency', util.base_model_fields.IntegerField(default=1, help_text='相同任务允许同时运行的最大实例数（仅在【自身可并发】开启时生效），至少为 1。', verbose_name='自身最大并发数')),
+                ('self_call_interval', util.base_model_fields.IntegerField(default=0, help_text='相同任务不可并发时，两次调用之间必须间隔的最小秒数（仅在【自身可并发】关闭时生效）。', verbose_name='自身调用最小间隔(秒)')),
+                ('data_start_date', util.base_model_fields.DateField(blank=True, null=True, verbose_name='数据开始日期')),
+                ('data_source', util.base_model_fields.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='lysync.syncdatasource', verbose_name='数据源')),
+            ],
+            options={
+                'verbose_name': '数据导出任务',
+            },
+            bases=(models.Model, util.basemodel.JSONBaseMixin, util.basemodel.ModefyMixin),
+        ),
+        migrations.CreateModel(
+            name='SyncModelType',
+            fields=[
+                ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('is_active', util.base_model_fields.BooleanField(choices=[(True, '可用'), (False, '禁用')], default=True, verbose_name='是否可用')),
+                ('create_time', util.base_model_fields.DateTimeField(db_index=True, default=django.utils.timezone.now, verbose_name='创建时间')),
+                ('update_time', util.base_model_fields.DateTimeField(auto_now=True, db_index=True, verbose_name='更新时间')),
+                ('name', util.base_model_fields.CharField(db_index=True, max_length=50, verbose_name='模型名称')),
+                ('app_label', util.base_model_fields.CharField(db_index=True, max_length=100, verbose_name='应用标签')),
+                ('model', util.base_model_fields.CharField(db_index=True, max_length=100, verbose_name='模型名')),
+                ('sorted', util.base_model_fields.IntegerField(db_index=True, default=0, verbose_name='数据处理排序')),
+                ('process_list', util.base_model_fields.JSONField(default=list, verbose_name='处理函数列表')),
+            ],
+            options={
+                'verbose_name': '数据模型',
+                'ordering': ['sorted'],
+            },
+            bases=(models.Model, util.basemodel.JSONBaseMixin, util.basemodel.ModefyMixin),
+        ),
+        migrations.CreateModel(
+            name='SyncModelField',
+            fields=[
+                ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('is_active', util.base_model_fields.BooleanField(choices=[(True, '可用'), (False, '禁用')], default=True, verbose_name='是否可用')),
+                ('create_time', util.base_model_fields.DateTimeField(db_index=True, default=django.utils.timezone.now, verbose_name='创建时间')),
+                ('update_time', util.base_model_fields.DateTimeField(auto_now=True, db_index=True, verbose_name='更新时间')),
+                ('name', util.base_model_fields.CharField(db_index=True, max_length=50, verbose_name='字段名')),
+                ('field_name', util.base_model_fields.CharField(db_index=True, max_length=50, verbose_name='字段名')),
+                ('is_unique', util.base_model_fields.BooleanField(default=False, help_text='可以配置多个字段，用于唯一标识数据行', verbose_name='是否唯一字段')),
+                ('is_datetime_index', util.base_model_fields.BooleanField(default=False, help_text='只能配置一个字段，用户数据同步时确定时间范围', verbose_name='是否日期时间索引')),
+                ('process_list', util.base_model_fields.JSONField(default=list, verbose_name='处理函数列表')),
+                ('is_import_column', util.base_model_fields.BooleanField(choices=[(True, '导入'), (False, '非导入')], default=True, verbose_name='是否为导入列')),
+                ('model_type', util.base_model_fields.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='lysync.syncmodeltype', verbose_name='数据模型')),
+            ],
+            options={
+                'verbose_name': '数据字段',
+            },
+            bases=(models.Model, util.basemodel.JSONBaseMixin, util.basemodel.ModefyMixin),
+        ),
+        migrations.CreateModel(
+            name='SyncExportTaskHistory',
+            fields=[
+                ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('is_active', util.base_model_fields.BooleanField(choices=[(True, '可用'), (False, '禁用')], default=True, verbose_name='是否可用')),
+                ('create_time', util.base_model_fields.DateTimeField(db_index=True, default=django.utils.timezone.now, verbose_name='创建时间')),
+                ('update_time', util.base_model_fields.DateTimeField(auto_now=True, db_index=True, verbose_name='更新时间')),
+                ('sql', util.base_model_fields.TextField(verbose_name='SQL')),
+                ('export_task', util.base_model_fields.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='lysync.syncexporttask', verbose_name='任务')),
+            ],
+            options={
+                'verbose_name': '数据导出任务历史',
+                'ordering': ['id'],
+            },
+            bases=(models.Model, util.basemodel.JSONBaseMixin, util.basemodel.ModefyMixin),
+        ),
+        migrations.AddField(
+            model_name='syncexporttask',
+            name='model_type',
+            field=util.base_model_fields.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='lysync.syncmodeltype', verbose_name='数据模型'),
+        ),
+        migrations.CreateModel(
+            name='SyncExportRecord',
+            fields=[
+                ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('is_active', util.base_model_fields.BooleanField(choices=[(True, '可用'), (False, '禁用')], default=True, verbose_name='是否可用')),
+                ('create_time', util.base_model_fields.DateTimeField(db_index=True, default=django.utils.timezone.now, verbose_name='创建时间')),
+                ('update_time', util.base_model_fields.DateTimeField(auto_now=True, db_index=True, verbose_name='更新时间')),
+                ('org_id', util.base_model_fields.IntegerField(verbose_name='所属组织')),
+                ('original_sql', util.base_model_fields.TextField(verbose_name='原始 SQL')),
+                ('exec_sql', util.base_model_fields.TextField(verbose_name='执行 SQL')),
+                ('export_status', util.base_model_fields.CharField(choices=[('checking', '检查中'), ('running', '运行中'), ('success', '成功'), ('fail', '失败'), ('stopping', '停止中'), ('stopped', '已停止'), ('wait', '等待中')], max_length=50, verbose_name='导出状态')),
+                ('check_status', util.base_model_fields.CharField(blank=True, choices=[('complete', '完成'), ('wait', '等待检查'), ('checking', '检查中')], max_length=50, null=True, verbose_name='检查状态')),
+                ('message', util.base_model_fields.TextField(verbose_name='导出消息')),
+                ('data_date', util.base_model_fields.DateField(blank=True, null=True, verbose_name='数据日期')),
+                ('export_unique_hash_list', util.base_model_fields.JSONField(blank=True, null=True, verbose_name='导出数据唯一标识列表')),
+                ('export_count', util.base_model_fields.IntegerField(default=0, verbose_name='导出数据数量')),
+                ('export_total_count', util.base_model_fields.IntegerField(default=0, help_text='导出数据总数,包含补充数据量', verbose_name='导出数据总数')),
+                ('last_check_message', util.base_model_fields.TextField(blank=True, editable=False, null=True, verbose_name='最后补录消息')),
+                ('last_check_record_id', util.base_model_fields.IntegerField(blank=True, editable=False, null=True, verbose_name='最后补录记录ID')),
+                ('success_check_record_count', util.base_model_fields.IntegerField(default=0, verbose_name='成功补录数量')),
+                ('last_check_time', util.base_model_fields.DateTimeField(blank=True, editable=False, null=True, verbose_name='最后补录时间')),
+                ('export_task', util.base_model_fields.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='lysync.syncexporttask', verbose_name='任务')),
+            ],
+            options={
+                'verbose_name': '数据导出记录',
+                'ordering': ['id'],
+            },
+            bases=(models.Model, util.basemodel.JSONBaseMixin, util.basemodel.ModefyMixin),
+        ),
+        migrations.CreateModel(
+            name='SyncExportCheckRecord',
+            fields=[
+                ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('is_active', util.base_model_fields.BooleanField(choices=[(True, '可用'), (False, '禁用')], default=True, verbose_name='是否可用')),
+                ('create_time', util.base_model_fields.DateTimeField(db_index=True, default=django.utils.timezone.now, verbose_name='创建时间')),
+                ('update_time', util.base_model_fields.DateTimeField(auto_now=True, db_index=True, verbose_name='更新时间')),
+                ('org_id', util.base_model_fields.IntegerField(verbose_name='所属组织')),
+                ('original_sql', util.base_model_fields.TextField(verbose_name='原始 SQL')),
+                ('exec_sql', util.base_model_fields.TextField(verbose_name='执行 SQL')),
+                ('export_status', util.base_model_fields.CharField(choices=[('checking', '检查中'), ('running', '运行中'), ('success', '成功'), ('fail', '失败'), ('stopping', '停止中'), ('stopped', '已停止'), ('wait', '等待中')], max_length=50, verbose_name='导出状态')),
+                ('message', util.base_model_fields.TextField(verbose_name='导出消息')),
+                ('export_unique_hash_list', util.base_model_fields.JSONField(blank=True, null=True, verbose_name='导出数据唯一标识列表')),
+                ('export_count', util.base_model_fields.IntegerField(default=0, verbose_name='导出数据数量')),
+                ('export_record', util.base_model_fields.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='lysync.syncexportrecord', verbose_name='导出记录')),
+                ('export_task', util.base_model_fields.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='lysync.syncexporttask', verbose_name='任务')),
+            ],
+            options={
+                'verbose_name': '数据导出补充记录',
+                'ordering': ['id'],
+            },
+            bases=(models.Model, util.basemodel.JSONBaseMixin, util.basemodel.ModefyMixin),
+        ),
+        migrations.CreateModel(
+            name='SyncDataFingerprint',
+            fields=[
+                ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('is_active', util.base_model_fields.BooleanField(choices=[(True, '可用'), (False, '禁用')], default=True, verbose_name='是否可用')),
+                ('create_time', util.base_model_fields.DateTimeField(db_index=True, default=django.utils.timezone.now, verbose_name='创建时间')),
+                ('update_time', util.base_model_fields.DateTimeField(auto_now=True, db_index=True, verbose_name='更新时间')),
+                ('org_id', util.base_model_fields.IntegerField(verbose_name='所属组织')),
+                ('origin_recode_unique_hash', util.base_model_fields.CharField(db_index=True, max_length=32, verbose_name='原始数据唯一标识')),
+                ('fingerprint', util.base_model_fields.CharField(max_length=32, verbose_name='数据指纹')),
+                ('export_task', util.base_model_fields.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='lysync.syncexporttask', verbose_name='任务')),
+            ],
+            options={
+                'verbose_name': '数据指纹',
+                'unique_together': {('export_task', 'origin_recode_unique_hash')},
+            },
+            bases=(models.Model, util.basemodel.JSONBaseMixin, util.basemodel.ModefyMixin),
+        ),
+    ]

lysync/migrations/0002_alter_syncexporttask_explain_json.py

@@ -0,0 +1,19 @@
+# Generated by Django 3.2.25 on 2026-07-11 17:54
+
+from django.db import migrations
+import util.base_model_fields
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('lysync', '0001_initial'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='syncexporttask',
+            name='explain_json',
+            field=util.base_model_fields.JSONField(blank=True, editable=False, null=True, verbose_name='SQL性能结果'),
+        ),
+    ]