ddlkit 0.1.0__py3-none-any.whl

ddlkit/keywords.py

@@ -0,0 +1,82 @@
+"""关键字表：全部在运行时从 sqlglot 方言派生，不硬编码。
+
+为什么必须派生：
+sqlglot 的 Tokenizer 用 trie 把部分关键字**合并成单个 token**，
+且每个方言的合并集合不同（实测 oracle 18 / mysql 21 / clickhouse 16 / hive 23）。
+例如 ``PRIMARY KEY (ID)`` 只产出 2 个 token：``PRIMARY KEY`` 和 ``(``。
+若按 ``"PRIMARY"`` 去匹配，约束会被误判为普通列。
+"""
+from __future__ import annotations
+
+from functools import lru_cache
+
+from sqlglot.dialects.dialect import Dialect
+
+
+@lru_cache(maxsize=None)
+def keyword_table(dialect: str) -> frozenset[str]:
+    """该方言 Tokenizer 的全部关键字（大写）。"""
+    cls = Dialect.get_or_raise(dialect).tokenizer_class
+    keywords = cls.KEYWORDS if isinstance(cls, type) else type(cls).KEYWORDS
+    return frozenset(k.upper() for k in keywords)
+
+
+@lru_cache(maxsize=None)
+def multiword_keywords(dialect: str) -> frozenset[str]:
+    """会被合并成单个 token 的多词关键字，如 ``PRIMARY KEY`` / ``ORDER BY``。"""
+    return frozenset(k for k in keyword_table(dialect) if " " in k)
+
+
+# CREATE 语句里允许出现在表名之前的修饰词，全部丢弃
+DECLARATION_MODIFIERS: frozenset[str] = frozenset({
+    "CREATE", "OR", "REPLACE", "TABLE", "IF", "NOT", "EXISTS",
+    "GLOBAL", "TEMPORARY", "TEMP", "LOCAL", "EXTERNAL", "VOLATILE",
+    "TRANSIENT", "DYNAMIC", "UNSET", "SET", "OPTIONS", "ICEBERG",
+})
+
+# 表体括号之外、会开启一个新"尾部子句"的起始标记。
+#
+# 注意：这里写的是 token **实际呈现的文本**（大写），因此多词关键字必须写合并形式
+#       （ORDER BY 而不是 ORDER），单词关键字写原样。
+# 注意：**故意不收录裸 ``AS``**——Hive 的 ``STORED AS`` 是两个独立 token
+#       （STORED / AS），收录 AS 会把 "STORED AS" 拆断。
+CLAUSE_STARTERS: frozenset[str] = frozenset({
+    # 多词（已合并）
+    "ORDER BY", "PARTITION BY", "PARTITIONED BY", "CLUSTER BY",
+    "DISTRIBUTE BY", "SORT BY", "PRIMARY KEY", "FOREIGN KEY",
+    # 单词
+    "ENGINE", "SETTINGS", "STORAGE", "TBLPROPERTIES", "SERDEPROPERTIES",
+    "COMMENT", "LOCATION", "ORGANIZATION", "REPLICA_NUM", "BLOCK_SIZE",
+    "COMPRESSION", "TABLET_SIZE", "PCTFREE", "USE_BLOOM_FILTER",
+    "ENABLE_MACRO_BLOCK_BLOOM_FILTER", "LOCALITY", "TABLEGROUP",
+    "CHARSET", "CHARACTER SET", "COLLATE", "TABLESPACE", "DATA_RETENTION",
+    "ROW", "STORED", "INPUTFORMAT", "OUTPUTFORMAT",
+    "DEFAULT", "WITH", "ON",
+})
+
+# 列定义区的表级约束起始标记
+CONSTRAINT_STARTERS: frozenset[str] = frozenset({
+    "PRIMARY KEY", "PRIMARY", "UNIQUE", "UNIQUE KEY", "KEY", "INDEX",
+    "FOREIGN KEY", "FOREIGN", "CHECK", "CONSTRAINT",
+    "NOT", "CLUSTER",  # 达梦 NOT CLUSTER PRIMARY KEY(...) / CLUSTER PRIMARY KEY(...)
+    "PARTITION", "PARTITION BY", "SUBPARTITION",
+})
+
+# 列属性关键字：出现即表示"类型已结束"或"DEFAULT 取值已结束"
+COLUMN_ATTRS: frozenset[str] = frozenset({
+    "NOT", "NULL", "DEFAULT", "COMMENT", "IDENTITY", "AUTO_INCREMENT",
+    "AUTOINCREMENT", "PRIMARY", "PRIMARY KEY", "UNIQUE", "REFERENCES",
+    "CHECK", "COLLATE", "GENERATED", "ENCODE", "CODEC", "TTL",
+    "CLUSTER", "STORAGE", "ON", "CONSTRAINT", "VISIBLE", "INVISIBLE",
+    "SORTKEY", "DISTKEY", "MASKING", "ENCRYPT",
+})
+
+# DEFAULT 取值的终止标记。
+# 关键修正：**排除 NULL**——否则 ``DEFAULT NULL`` 会把 NULL 当终止符，
+# 导致 default_raw 变成空串（实测源文件里这样的写法有 4449 处）。
+DEFAULT_TERMINATORS: frozenset[str] = COLUMN_ATTRS - {"NULL"}
+
+# 可以跟在类型名之后、仍属于类型的一部分（非别名化的多词类型）
+TYPE_TRAILERS: frozenset[str] = frozenset({
+    "UNSIGNED", "SIGNED", "ZEROFILL", "PRECISION", "VARYING",
+})

ddlkit/lexer.py

@@ -0,0 +1,113 @@
+"""词法层：调用 sqlglot 的 Tokenizer，并保证"原文可无损取回"。
+
+为什么不用 parser：
+parser 会挂死（达梦 ``NOT CLUSTER PRIMARY KEY`` 导致无限回溯 / OOM），
+并且会主动改写它"认识"的语法（``TINYINT`` -> ``SMALLINT``）。
+tokenizer 只做词法，不猜语义，这两类问题都不存在。
+
+本模块解决的关键陷阱：
+``Token.text`` **不是原文**——字符串字面量的引号和标识符的引号/反引号都会被剥掉。
+实测 100218 列里 ``quoted=True`` 的判据因此恒为 False，32091 个字符串字面量的引号全部丢失。
+解法：用 ``sql[token.start : token.end + 1]`` 切原串（``end`` 是**闭区间**，必须 +1）。
+"""
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+import sqlglot
+from sqlglot.tokens import Token, TokenType
+
+# 不参与结构分析、但在原文切片中"天然存在"的 token
+SKIP_TYPES: frozenset[TokenType] = frozenset({TokenType.SPACE, TokenType.BREAK})
+
+
+@dataclass(slots=True)
+class Lexed:
+    """一段 SQL 的词法结果，携带原文以支持无损切片。"""
+
+    sql: str
+    tokens: list[Token]
+
+    def __len__(self) -> int:
+        return len(self.tokens)
+
+    def tok(self, i: int) -> Token:
+        return self.tokens[i]
+
+    def text(self, i: int) -> str:
+        """token 的归一化文本（引号已剥）。"""
+        return self.tokens[i].text
+
+    def norm(self, i: int) -> str:
+        """token 文本大写，用于与关键字集合比较。"""
+        return self.tokens[i].text.upper()
+
+    def raw(self, i: int) -> str:
+        """单个 token 的**原始文本**（含引号）。"""
+        t = self.tokens[i]
+        return self.sql[t.start : t.end + 1]
+
+    def span(self, i: int, j: int) -> str:
+        """[i, j] 闭区间 token 覆盖的**原始片段**（保留原始空白与引号）。
+
+        直接按首尾偏移切原串，而不是把 ``text`` 拼起来——后者会丢引号和原始间距。
+        """
+        if i > j:
+            return ""
+        return self.sql[self.tokens[i].start : self.tokens[j].end + 1]
+
+    def line_of(self, i: int) -> int:
+        return self.tokens[i].line
+
+    def col_of(self, i: int) -> int:
+        return self.tokens[i].col
+
+
+def lex(sql: str, dialect: str) -> Lexed:
+    """词法分析。注意 ``sqlglot.tokenize`` 的签名是 ``(sql, read=None, dialect=None)``。"""
+    tokens = [t for t in sqlglot.tokenize(sql, dialect=dialect) if t.token_type not in SKIP_TYPES]
+    return Lexed(sql=sql, tokens=tokens)
+
+
+def find_matching(tokens: list[Token], start: int, end: int) -> int:
+    """给定 ``(`` 的下标，返回配对 ``)`` 的下标；找不到返回 -1。"""
+    depth = 0
+    for i in range(start, end + 1):
+        txt = tokens[i].text
+        if txt == "(":
+            depth += 1
+        elif txt == ")":
+            depth -= 1
+            if depth == 0:
+                return i
+    return -1
+
+
+def split_top(tokens: list[Token], start: int, end: int) -> list[tuple[int, int]]:
+    """按顶层逗号切分，返回若干 (起, 止) 下标区间。"""
+    parts: list[tuple[int, int]] = []
+    cur = start
+    depth = 0
+    for i in range(start, end + 1):
+        txt = tokens[i].text
+        if txt in ("(", "["):
+            depth += 1
+        elif txt in (")", "]"):
+            depth -= 1
+        elif txt == "," and depth == 0:
+            if i > cur:
+                parts.append((cur, i - 1))
+            cur = i + 1
+    if cur <= end:
+        parts.append((cur, end))
+    return parts
+
+
+def unquote(s: str) -> str:
+    """去掉标识符 / 字符串的包裹引号。"""
+    if len(s) >= 2 and s[0] == s[-1]:
+        if s[0] in ('"', "`", "'"):
+            return s[1:-1]
+        if s[0] == "[" and s[-1] == "]":
+            return s[1:-1]
+    return s

ddlkit/model.py

@@ -0,0 +1,95 @@
+"""统一信息模型（UIM）——本包对外的唯一数据契约。
+
+设计原则：
+1. 双轨字段：``*_raw`` 保留原文（含引号、原始类型名），归一化字段供规则比较。
+2. 方言特性一律进 ``extras``，**结构化保存而非丢弃**。
+3. 绝不向外暴露 sqlglot 的 Expression / Token，避免上游升级连带破坏下游。
+"""
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+
+
+@dataclass(slots=True)
+class SourceRef:
+    """来源定位，供报告回溯到文件行。"""
+
+    path: str | None = None
+    line: int = 0
+    encoding: str = ""
+    fmt: str = ""
+
+
+@dataclass(slots=True)
+class Column:
+    name: str = ""                       # 去引号后的列名
+    name_raw: str = ""                   # 原文（含引号）："ID" / `id` / ID
+    quoted: bool = False                 # 是否被引号包裹（书写层信息）
+    type_name: str = ""                  # 首个类型词，大写：VARCHAR / TINYINT / NULLABLE
+    type_raw: str = ""                   # 完整原始类型文本：VARCHAR2(50) / DEC(20,0) / Nullable(String)
+    type_args_raw: str | None = None     # 括号部分原文：(50) / (10,2) / (String)
+    nullable: bool | None = None         # 派生字段；None = 未显式声明
+    default_raw: str | None = None       # DEFAULT 之后的值原文
+    comment: str | None = None
+    extras: dict[str, str] = field(default_factory=dict)
+    line: int = 0
+    col: int = 0
+
+
+@dataclass(slots=True)
+class Constraint:
+    kind: str = ""                       # PRIMARY KEY / UNIQUE / FOREIGN KEY / CHECK / KEY / INDEX
+    name: str | None = None
+    name_raw: str | None = None
+    columns: list[str] = field(default_factory=list)
+    clustered: bool | None = None        # 达梦 CLUSTER / NOT CLUSTER；None = 未声明
+    extras: dict[str, str] = field(default_factory=dict)
+    raw: str = ""
+    line: int = 0
+
+
+@dataclass(slots=True)
+class Index:
+    name: str | None = None
+    name_raw: str | None = None
+    kind: str = "INDEX"                  # INDEX / UNIQUE INDEX / KEY
+    columns: list[str] = field(default_factory=list)
+    extras: dict[str, str] = field(default_factory=dict)
+    raw: str = ""
+    line: int = 0
+
+
+@dataclass(slots=True)
+class Table:
+    dialect: str = ""
+    catalog: str | None = None
+    schema: str | None = None
+    name: str = ""
+    name_raw: str = ""
+    quoted: bool = False
+    temporary: bool = False
+    columns: list[Column] = field(default_factory=list)
+    constraints: list[Constraint] = field(default_factory=list)
+    indexes: list[Index] = field(default_factory=list)
+    comment: str | None = None
+    extras: dict[str, str] = field(default_factory=dict)
+    source: SourceRef = field(default_factory=SourceRef)
+    warnings: list[str] = field(default_factory=list)
+
+    @property
+    def qualified_name(self) -> str:
+        return ".".join(x for x in (self.catalog, self.schema, self.name) if x)
+
+    def column(self, name: str) -> Column | None:
+        upper = name.upper()
+        for c in self.columns:
+            if c.name.upper() == upper or c.name_raw.upper() == upper:
+                return c
+        return None
+
+
+@dataclass(slots=True)
+class ParseResult:
+    tables: list[Table] = field(default_factory=list)
+    warnings: list[str] = field(default_factory=list)
+    unsupported: list[str] = field(default_factory=list)

ddlkit/source.py

@@ -0,0 +1,274 @@
+"""导出格式识别与语句切分。
+
+实测这批导出文件**实际有 5 种格式，不是 4 种**（按库划分会漏掉第 5 种）：
+
+======  ==========================  ==========================================
+格式    特征                        切分方式
+======  ==========================  ==========================================
+beeline ``| DDL |`` 表格 + 表头      剥边框后按锚点切
+quoted 每行一条、整行被双引号包裹    逐行，剥首尾引号（**无分号**）
+escaped 每行一条、内部换行是字面\\n  逐行，反转义（**分号在语句内部，不能按分号切**）
+anchor  原生脚本但**全文 0 个分号**  按 CREATE/COMMENT ON 锚点切
+semicolon 常规脚本                   按顶层分号切（注释/引号/括号感知）
+======  ==========================  ==========================================
+
+踩过的坑：
+* ``Hive/可靠性_*.sql``（547 表）是 **anchor** 格式，全文分号数 = 0。
+  按分号切会把整个文件并成 1 条语句。
+* OB 文件里语句内部含 57 个分号，按分号切会误切。
+* ClickHouse 无分号，按分号切同样并成 1 条。
+"""
+from __future__ import annotations
+
+import bisect
+import re
+from dataclasses import dataclass
+
+# ---------------------------------------------------------------- 语句头识别
+
+_CREATE_TABLE = re.compile(
+    r"^\s*CREATE\s+(?:OR\s+REPLACE\s+)?"
+    r"(?:GLOBAL\s+TEMPORARY\s+|LOCAL\s+TEMPORARY\s+|TEMPORARY\s+|"
+    r"EXTERNAL\s+|VOLATILE\s+|TRANSIENT\s+)?TABLE\b",
+    re.IGNORECASE,
+)
+_CREATE_INDEX = re.compile(
+    r"^\s*CREATE\s+(?:OR\s+REPLACE\s+)?"
+    r"(?:UNIQUE\s+|BITMAP\s+|CLUSTERED\s+|NONCLUSTERED\s+|FULLTEXT\s+|SPATIAL\s+)?"
+    r"(?:GLOBAL\s+|LOCAL\s+)?(?:INDEX|KEY)\b",
+    re.IGNORECASE,
+)
+_COMMENT_ON = re.compile(r"^\s*COMMENT\s+ON\b", re.IGNORECASE)
+
+# anchor 格式的语句起点。只认这三类——不能笼统地用 `^CREATE`，
+# 否则 CREATE VIEW / CREATE SEQUENCE 会变成锚点并把后续语句吞进去。
+_ANCHOR = re.compile(
+    r"^\s*(?:"
+    r"CREATE\s+(?:OR\s+REPLACE\s+)?"
+    r"(?:GLOBAL\s+TEMPORARY\s+|LOCAL\s+TEMPORARY\s+|TEMPORARY\s+|EXTERNAL\s+|VOLATILE\s+|TRANSIENT\s+)?TABLE\b"
+    r"|CREATE\s+(?:OR\s+REPLACE\s+)?(?:UNIQUE\s+)?(?:INDEX|KEY)\b"
+    r"|COMMENT\s+ON\b"
+    r")",
+    re.IGNORECASE,
+)
+
+
+@dataclass(slots=True)
+class Statement:
+    sql: str
+    line: int  # 1-based，语句在原始文件中的起始行
+    kind: str  # create_table / create_index / comment_on / other
+
+
+def classify(sql: str) -> str:
+    if _CREATE_TABLE.match(sql):
+        return "create_table"
+    if _CREATE_INDEX.match(sql):
+        return "create_index"
+    if _COMMENT_ON.match(sql):
+        return "comment_on"
+    return "other"
+
+
+# ---------------------------------------------------------------- 工具
+
+
+def _line_starts(text: str) -> list[int]:
+    """每一行起始偏移，供 offset -> 行号 换算。"""
+    pos = [0]
+    i = text.find("\n")
+    while i >= 0:
+        pos.append(i + 1)
+        i = text.find("\n", i + 1)
+    return pos
+
+
+def _line_no(starts: list[int], offset: int) -> int:
+    return bisect.bisect_right(starts, offset)
+
+
+def strip_leading(text: str) -> tuple[str, int]:
+    """剥掉开头的空白与整行/整块注释，返回 (剩余文本, 剥掉的字符数)。"""
+    off = 0
+    n = len(text)
+    while off < n:
+        m = re.match(r"\s*", text[off:])
+        assert m is not None
+        k = off + m.end()
+        rest = text[k:]
+        if rest.startswith("--") or rest.startswith("#"):
+            nl = rest.find("\n")
+            if nl < 0:
+                return "", n
+            off = k + nl + 1
+            continue
+        if rest.startswith("/*"):
+            end = rest.find("*/")
+            if end < 0:
+                return "", n
+            off = k + end + 2
+            continue
+        return rest, k
+    return "", n
+
+
+def _scan_semicolons(text: str) -> list[tuple[int, int]]:
+    """按顶层分号切分，返回 (起, 止) 偏移对。
+
+    状态机感知：``'`` / ``"`` / 反引号字符串（含 ``''`` 双写与 ``\\`` 转义）、
+    ``--`` 行注释、``/* */`` 块注释、括号深度。
+    注释感知是必需的——注释里的括号或引号会污染状态，导致整个文件再也不能切分。
+    """
+    out: list[tuple[int, int]] = []
+    n = len(text)
+    i = 0
+    start = 0
+    depth = 0
+    while i < n:
+        ch = text[i]
+        if ch == "-" and text.startswith("--", i):
+            j = text.find("\n", i)
+            i = n if j < 0 else j
+            continue
+        if ch == "/" and text.startswith("/*", i):
+            j = text.find("*/", i + 2)
+            i = n if j < 0 else j + 2
+            continue
+        if ch in ("'", '"', "`"):
+            quote = ch
+            i += 1
+            while i < n:
+                c = text[i]
+                if c == "\\" and i + 1 < n:
+                    i += 2
+                    continue
+                if c == quote:
+                    if i + 1 < n and text[i + 1] == quote:
+                        i += 2
+                        continue
+                    i += 1
+                    break
+                i += 1
+            continue
+        if ch == "(":
+            depth += 1
+        elif ch == ")":
+            depth = max(0, depth - 1)
+        elif ch == ";" and depth == 0:
+            out.append((start, i))
+            start = i + 1
+        i += 1
+    if start < n:
+        out.append((start, n))
+    return out
+
+
+def _emit(text: str, spans: list[tuple[int, int]], starts: list[int]) -> list[Statement]:
+    res: list[Statement] = []
+    for a, b in spans:
+        body, off = strip_leading(text[a:b])
+        if not body.strip():
+            continue
+        res.append(Statement(sql=body, line=_line_no(starts, a + off), kind=classify(body)))
+    return res
+
+
+# ---------------------------------------------------------------- 格式识别
+
+def detect(text: str) -> str:
+    """判定导出格式。顺序敏感，先判特异性最强的。"""
+    if "createtab_stmt" in text:
+        return "beeline"
+
+    escaped = text.count("\\n")
+    real = text.count("\n")
+    # 字面量 \n 明显多于真实换行 -> 该导出把逻辑换行做了转义
+    if escaped >= 3 and escaped > real:
+        return "escaped_lines"
+
+    first = next((ln.strip() for ln in text.splitlines() if ln.strip()), "")
+    if len(first) > 1 and first[0] == '"' and first[-1] == '"':
+        return "quoted_lines"
+
+    if text.count(";") == 0:
+        return "anchor"
+
+    return "semicolon"
+
+
+# ---------------------------------------------------------------- 各格式切分
+
+def _split_anchor(text: str, starts: list[int]) -> list[Statement]:
+    lines = text.splitlines(keepends=True)
+    offsets: list[int] = []
+    acc = 0
+    for ln in lines:
+        offsets.append(acc)
+        acc += len(ln)
+
+    anchors = [i for i, ln in enumerate(lines) if _ANCHOR.match(ln)]
+    if not anchors:
+        return []
+
+    spans: list[tuple[int, int]] = []
+    for k, idx in enumerate(anchors):
+        stop = anchors[k + 1] if k + 1 < len(anchors) else len(lines)
+        a = offsets[idx]
+        b = offsets[stop] if stop < len(lines) else len(text)
+        spans.append((a, b))
+    return _emit(text, spans, starts)
+
+
+def _split_beeline(text: str, starts: list[int]) -> list[Statement]:
+    """剥掉表格边框，保持行数不变（这样行号仍对应原文件），再按锚点切。"""
+    kept: list[str] = []
+    for ln in text.splitlines(keepends=True):
+        body = ln.strip()
+        if body.startswith("|"):
+            inner = body.strip("|").strip()
+            if inner.lower() == "createtab_stmt":
+                inner = ""
+            kept.append(inner + "\n")
+        else:
+            kept.append("\n")
+    return _split_anchor("".join(kept), _line_starts("".join(kept)))
+
+
+def _split_lines(text: str, starts: list[int], unescape: bool) -> list[Statement]:
+    res: list[Statement] = []
+    offset = 0
+    for idx, ln in enumerate(text.splitlines(keepends=True)):
+        body = ln.strip()
+        if body:
+            if body.startswith('"') and body.endswith('"') and len(body) > 1:
+                body = body[1:-1]
+            if unescape:
+                body = body.replace("\\n", "\n")
+            body, off = strip_leading(body)
+            if body.strip():
+                res.append(
+                    Statement(
+                        sql=body,
+                        line=_line_no(starts, offset + off),
+                        kind=classify(body),
+                    )
+                )
+        offset += len(ln)
+    return res
+
+
+# ---------------------------------------------------------------- 对外入口
+
+def split(text: str, fmt: str | None = None) -> list[Statement]:
+    """把整份导出文本切成语句列表。``fmt=None`` 时自动识别。"""
+    fmt = fmt or detect(text)
+    starts = _line_starts(text)
+    if fmt == "beeline":
+        return _split_beeline(text, starts)
+    if fmt == "quoted_lines":
+        return _split_lines(text, starts, unescape=False)
+    if fmt == "escaped_lines":
+        return _split_lines(text, starts, unescape=True)
+    if fmt == "anchor":
+        return _split_anchor(text, starts)
+    return _emit(text, _scan_semicolons(text), starts)