ddlkit 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ddlkit-0.1.0/LICENSE +21 -0
- ddlkit-0.1.0/PKG-INFO +147 -0
- ddlkit-0.1.0/README.md +119 -0
- ddlkit-0.1.0/pyproject.toml +76 -0
- ddlkit-0.1.0/src/ddlkit/__init__.py +24 -0
- ddlkit-0.1.0/src/ddlkit/api.py +110 -0
- ddlkit-0.1.0/src/ddlkit/comments.py +72 -0
- ddlkit-0.1.0/src/ddlkit/encoding.py +34 -0
- ddlkit-0.1.0/src/ddlkit/extract.py +398 -0
- ddlkit-0.1.0/src/ddlkit/keywords.py +82 -0
- ddlkit-0.1.0/src/ddlkit/lexer.py +113 -0
- ddlkit-0.1.0/src/ddlkit/model.py +95 -0
- ddlkit-0.1.0/src/ddlkit/source.py +274 -0
- ddlkit-0.1.0/tests/test_smoke.py +280 -0
ddlkit-0.1.0/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 yangyang
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
ddlkit-0.1.0/PKG-INFO
ADDED
|
@@ -0,0 +1,147 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: ddlkit
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: 多方言 DDL 文本解析器:无损保真地提取表、列、约束、索引与方言特性
|
|
5
|
+
Project-URL: Homepage, https://github.com/YOUR-ORG/ddl-kit
|
|
6
|
+
Project-URL: Repository, https://github.com/YOUR-ORG/ddl-kit
|
|
7
|
+
Project-URL: Issues, https://github.com/YOUR-ORG/ddl-kit/issues
|
|
8
|
+
Project-URL: Changelog, https://github.com/YOUR-ORG/ddl-kit/blob/main/CHANGELOG.md
|
|
9
|
+
Author: yangyang
|
|
10
|
+
License-Expression: MIT
|
|
11
|
+
License-File: LICENSE
|
|
12
|
+
Keywords: audit,clickhouse,dameng,ddl,governance,hive,oceanbase,parser,sql
|
|
13
|
+
Classifier: Development Status :: 3 - Alpha
|
|
14
|
+
Classifier: Intended Audience :: Developers
|
|
15
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
16
|
+
Classifier: Operating System :: OS Independent
|
|
17
|
+
Classifier: Programming Language :: Python :: 3
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
21
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
22
|
+
Classifier: Topic :: Database
|
|
23
|
+
Classifier: Topic :: Software Development :: Quality Assurance
|
|
24
|
+
Classifier: Typing :: Typed
|
|
25
|
+
Requires-Python: >=3.10
|
|
26
|
+
Requires-Dist: sqlglot<31,>=30.18
|
|
27
|
+
Description-Content-Type: text/markdown
|
|
28
|
+
|
|
29
|
+
# ddlkit
|
|
30
|
+
|
|
31
|
+
多方言 DDL 文本解析器。只做一件事:**DDL 文本 → 结构化数据**,不含规则引擎。
|
|
32
|
+
|
|
33
|
+
面向的场景是**建表规范审计**:校验表名/列名/索引名的书写形态、长度、注释是否齐全、
|
|
34
|
+
类型是否符合规范。这类审计对"保真"的要求极高——解析器**改写**或**丢弃**任何一处,
|
|
35
|
+
规则就会判错,而判错是静默的,比报错更危险。
|
|
36
|
+
|
|
37
|
+
## 核心特征
|
|
38
|
+
|
|
39
|
+
| 特征 | 说明 |
|
|
40
|
+
|---|---|
|
|
41
|
+
| **不丢原文** | 每个字段双轨:`*_raw` 保留原文(含引号、原始类型名),归一化字段供比较 |
|
|
42
|
+
| **不丢方言** | 方言特性(达梦 `STORAGE`、CK `ENGINE`/`ORDER BY`、Hive `TBLPROPERTIES`、OB `REPLICA_NUM`…)全部结构化进 `extras`,**不做摘除** |
|
|
43
|
+
| **不挂死** | 只走 sqlglot 的 tokenizer,不碰 parser。达梦 `NOT CLUSTER PRIMARY KEY` 曾导致 parser 无限回溯 OOM |
|
|
44
|
+
| **不改写** | 绕过 parser + generator,因此 `TINYINT` 不会变成 `SMALLINT` |
|
|
45
|
+
| **格式自适应** | 自动识别 5 种导出格式(beeline / 双引号包裹逐行 / `\n` 转义逐行 / 无分号锚点 / 常规脚本) |
|
|
46
|
+
| **编码自适应** | 逐文件探测 UTF-8 / UTF-8-BOM / GB18030(同一批导出文件里可能混存) |
|
|
47
|
+
| **零额外依赖** | 只依赖纯 Python 版 `sqlglot` |
|
|
48
|
+
|
|
49
|
+
## 安装
|
|
50
|
+
|
|
51
|
+
```bash
|
|
52
|
+
pip install ddlkit
|
|
53
|
+
# 或
|
|
54
|
+
uv add ddlkit
|
|
55
|
+
```
|
|
56
|
+
|
|
57
|
+
> 依赖声明为 `sqlglot`(纯 Python 版)。**不要同时安装 `sqlglot[c]`**——它会把
|
|
58
|
+
> 编译产物覆盖到 `sqlglot/tokenizer_core`,本包未在该形态下验证。
|
|
59
|
+
|
|
60
|
+
## 快速开始
|
|
61
|
+
|
|
62
|
+
```python
|
|
63
|
+
from ddlkit import parse_file
|
|
64
|
+
|
|
65
|
+
res = parse_file("ddl/ANMS_ALARM.sql", dialect="dm")
|
|
66
|
+
|
|
67
|
+
for t in res.tables:
|
|
68
|
+
print(t.qualified_name, len(t.columns), t.comment)
|
|
69
|
+
for c in t.columns:
|
|
70
|
+
# 用 type_raw 判断类型,不要用归一化后的值——那会丢原始写法
|
|
71
|
+
print(" ", c.name_raw, c.type_raw, c.comment)
|
|
72
|
+
for k in t.constraints:
|
|
73
|
+
print(" PK?", k.kind, k.columns, k.clustered)
|
|
74
|
+
print(" 方言特性:", t.extras)
|
|
75
|
+
```
|
|
76
|
+
|
|
77
|
+
解析一段文本:
|
|
78
|
+
|
|
79
|
+
```python
|
|
80
|
+
from ddlkit import parse_ddl
|
|
81
|
+
|
|
82
|
+
res = parse_ddl(
|
|
83
|
+
'''CREATE TABLE `t` (
|
|
84
|
+
`ID` varchar(512) NOT NULL COMMENT '主键',
|
|
85
|
+
PRIMARY KEY (`ID`)
|
|
86
|
+
) REPLICA_NUM = 3 COMMENT = '示例表';''',
|
|
87
|
+
dialect="ob_mysql",
|
|
88
|
+
)
|
|
89
|
+
t = res.tables[0]
|
|
90
|
+
assert t.constraints[0].kind == "PRIMARY KEY"
|
|
91
|
+
assert t.extras["REPLICA_NUM"] == "REPLICA_NUM = 3"
|
|
92
|
+
```
|
|
93
|
+
|
|
94
|
+
## 支持的方言别名
|
|
95
|
+
|
|
96
|
+
| 业务库 | 可传的 `dialect` | 映射到 sqlglot 方言 |
|
|
97
|
+
|---|---|---|
|
|
98
|
+
| 达梦 | `dm` / `dameng` / `达梦` | `oracle` |
|
|
99
|
+
| ClickHouse | `ck` / `clickhouse` | `clickhouse` |
|
|
100
|
+
| OceanBase(MySQL 模式) | `ob` / `ob_mysql` / `oceanbase` | `mysql` |
|
|
101
|
+
| Hive | `hive` / `hive2` | `hive` |
|
|
102
|
+
|
|
103
|
+
其余任意 sqlglot 方言名也可直接传入(如 `postgres`、`snowflake`)。
|
|
104
|
+
|
|
105
|
+
## 数据模型
|
|
106
|
+
|
|
107
|
+
```
|
|
108
|
+
ParseResult
|
|
109
|
+
├── tables: list[Table]
|
|
110
|
+
│ ├── name / name_raw / quoted / schema / catalog / temporary
|
|
111
|
+
│ ├── columns: list[Column]
|
|
112
|
+
│ │ ├── name / name_raw / quoted
|
|
113
|
+
│ │ ├── type_name / type_raw / type_args_raw
|
|
114
|
+
│ │ ├── nullable / default_raw / comment
|
|
115
|
+
│ │ ├── extras # identity / collate / codec / primary_key …
|
|
116
|
+
│ │ └── line / col
|
|
117
|
+
│ ├── constraints: list[Constraint] # PRIMARY KEY / UNIQUE / FOREIGN KEY / CHECK / INDEX
|
|
118
|
+
│ ├── indexes: list[Index] # 独立 CREATE INDEX 语句,已按表名挂接
|
|
119
|
+
│ ├── comment
|
|
120
|
+
│ ├── extras # 方言特性,原样保存
|
|
121
|
+
│ ├── source # 文件 / 行号 / 编码 / 格式,供报告定位
|
|
122
|
+
│ └── warnings
|
|
123
|
+
├── warnings
|
|
124
|
+
└── unsupported # 无法当作建表语句处理的语句(如 CTAS)
|
|
125
|
+
```
|
|
126
|
+
|
|
127
|
+
完整字段语义见 `docs/详细设计.md`。
|
|
128
|
+
|
|
129
|
+
## 已知限制
|
|
130
|
+
|
|
131
|
+
* 只解析 DDL,不解析 DML / 存储过程 / PL-SQL 块。
|
|
132
|
+
* 只解析 `CREATE TABLE` / `CREATE INDEX` / `COMMENT ON`;其余语句进 `unsupported`/忽略。
|
|
133
|
+
* 全角字符长度、`CASE_SENSITIVE` 语义等**库侧语义**不在解析层处理,交由规则层。
|
|
134
|
+
* 单条语句内部的极端嵌套(Oracle `q'[...]'` 引号、`$$` 块)未覆盖。
|
|
135
|
+
|
|
136
|
+
## 开发
|
|
137
|
+
|
|
138
|
+
```bash
|
|
139
|
+
uv sync
|
|
140
|
+
uv run pytest -q
|
|
141
|
+
uv run ruff check src tests
|
|
142
|
+
uv run mypy
|
|
143
|
+
```
|
|
144
|
+
|
|
145
|
+
## 许可
|
|
146
|
+
|
|
147
|
+
MIT
|
ddlkit-0.1.0/README.md
ADDED
|
@@ -0,0 +1,119 @@
|
|
|
1
|
+
# ddlkit
|
|
2
|
+
|
|
3
|
+
多方言 DDL 文本解析器。只做一件事:**DDL 文本 → 结构化数据**,不含规则引擎。
|
|
4
|
+
|
|
5
|
+
面向的场景是**建表规范审计**:校验表名/列名/索引名的书写形态、长度、注释是否齐全、
|
|
6
|
+
类型是否符合规范。这类审计对"保真"的要求极高——解析器**改写**或**丢弃**任何一处,
|
|
7
|
+
规则就会判错,而判错是静默的,比报错更危险。
|
|
8
|
+
|
|
9
|
+
## 核心特征
|
|
10
|
+
|
|
11
|
+
| 特征 | 说明 |
|
|
12
|
+
|---|---|
|
|
13
|
+
| **不丢原文** | 每个字段双轨:`*_raw` 保留原文(含引号、原始类型名),归一化字段供比较 |
|
|
14
|
+
| **不丢方言** | 方言特性(达梦 `STORAGE`、CK `ENGINE`/`ORDER BY`、Hive `TBLPROPERTIES`、OB `REPLICA_NUM`…)全部结构化进 `extras`,**不做摘除** |
|
|
15
|
+
| **不挂死** | 只走 sqlglot 的 tokenizer,不碰 parser。达梦 `NOT CLUSTER PRIMARY KEY` 曾导致 parser 无限回溯 OOM |
|
|
16
|
+
| **不改写** | 绕过 parser + generator,因此 `TINYINT` 不会变成 `SMALLINT` |
|
|
17
|
+
| **格式自适应** | 自动识别 5 种导出格式(beeline / 双引号包裹逐行 / `\n` 转义逐行 / 无分号锚点 / 常规脚本) |
|
|
18
|
+
| **编码自适应** | 逐文件探测 UTF-8 / UTF-8-BOM / GB18030(同一批导出文件里可能混存) |
|
|
19
|
+
| **零额外依赖** | 只依赖纯 Python 版 `sqlglot` |
|
|
20
|
+
|
|
21
|
+
## 安装
|
|
22
|
+
|
|
23
|
+
```bash
|
|
24
|
+
pip install ddlkit
|
|
25
|
+
# 或
|
|
26
|
+
uv add ddlkit
|
|
27
|
+
```
|
|
28
|
+
|
|
29
|
+
> 依赖声明为 `sqlglot`(纯 Python 版)。**不要同时安装 `sqlglot[c]`**——它会把
|
|
30
|
+
> 编译产物覆盖到 `sqlglot/tokenizer_core`,本包未在该形态下验证。
|
|
31
|
+
|
|
32
|
+
## 快速开始
|
|
33
|
+
|
|
34
|
+
```python
|
|
35
|
+
from ddlkit import parse_file
|
|
36
|
+
|
|
37
|
+
res = parse_file("ddl/ANMS_ALARM.sql", dialect="dm")
|
|
38
|
+
|
|
39
|
+
for t in res.tables:
|
|
40
|
+
print(t.qualified_name, len(t.columns), t.comment)
|
|
41
|
+
for c in t.columns:
|
|
42
|
+
# 用 type_raw 判断类型,不要用归一化后的值——那会丢原始写法
|
|
43
|
+
print(" ", c.name_raw, c.type_raw, c.comment)
|
|
44
|
+
for k in t.constraints:
|
|
45
|
+
print(" PK?", k.kind, k.columns, k.clustered)
|
|
46
|
+
print(" 方言特性:", t.extras)
|
|
47
|
+
```
|
|
48
|
+
|
|
49
|
+
解析一段文本:
|
|
50
|
+
|
|
51
|
+
```python
|
|
52
|
+
from ddlkit import parse_ddl
|
|
53
|
+
|
|
54
|
+
res = parse_ddl(
|
|
55
|
+
'''CREATE TABLE `t` (
|
|
56
|
+
`ID` varchar(512) NOT NULL COMMENT '主键',
|
|
57
|
+
PRIMARY KEY (`ID`)
|
|
58
|
+
) REPLICA_NUM = 3 COMMENT = '示例表';''',
|
|
59
|
+
dialect="ob_mysql",
|
|
60
|
+
)
|
|
61
|
+
t = res.tables[0]
|
|
62
|
+
assert t.constraints[0].kind == "PRIMARY KEY"
|
|
63
|
+
assert t.extras["REPLICA_NUM"] == "REPLICA_NUM = 3"
|
|
64
|
+
```
|
|
65
|
+
|
|
66
|
+
## 支持的方言别名
|
|
67
|
+
|
|
68
|
+
| 业务库 | 可传的 `dialect` | 映射到 sqlglot 方言 |
|
|
69
|
+
|---|---|---|
|
|
70
|
+
| 达梦 | `dm` / `dameng` / `达梦` | `oracle` |
|
|
71
|
+
| ClickHouse | `ck` / `clickhouse` | `clickhouse` |
|
|
72
|
+
| OceanBase(MySQL 模式) | `ob` / `ob_mysql` / `oceanbase` | `mysql` |
|
|
73
|
+
| Hive | `hive` / `hive2` | `hive` |
|
|
74
|
+
|
|
75
|
+
其余任意 sqlglot 方言名也可直接传入(如 `postgres`、`snowflake`)。
|
|
76
|
+
|
|
77
|
+
## 数据模型
|
|
78
|
+
|
|
79
|
+
```
|
|
80
|
+
ParseResult
|
|
81
|
+
├── tables: list[Table]
|
|
82
|
+
│ ├── name / name_raw / quoted / schema / catalog / temporary
|
|
83
|
+
│ ├── columns: list[Column]
|
|
84
|
+
│ │ ├── name / name_raw / quoted
|
|
85
|
+
│ │ ├── type_name / type_raw / type_args_raw
|
|
86
|
+
│ │ ├── nullable / default_raw / comment
|
|
87
|
+
│ │ ├── extras # identity / collate / codec / primary_key …
|
|
88
|
+
│ │ └── line / col
|
|
89
|
+
│ ├── constraints: list[Constraint] # PRIMARY KEY / UNIQUE / FOREIGN KEY / CHECK / INDEX
|
|
90
|
+
│ ├── indexes: list[Index] # 独立 CREATE INDEX 语句,已按表名挂接
|
|
91
|
+
│ ├── comment
|
|
92
|
+
│ ├── extras # 方言特性,原样保存
|
|
93
|
+
│ ├── source # 文件 / 行号 / 编码 / 格式,供报告定位
|
|
94
|
+
│ └── warnings
|
|
95
|
+
├── warnings
|
|
96
|
+
└── unsupported # 无法当作建表语句处理的语句(如 CTAS)
|
|
97
|
+
```
|
|
98
|
+
|
|
99
|
+
完整字段语义见 `docs/详细设计.md`。
|
|
100
|
+
|
|
101
|
+
## 已知限制
|
|
102
|
+
|
|
103
|
+
* 只解析 DDL,不解析 DML / 存储过程 / PL-SQL 块。
|
|
104
|
+
* 只解析 `CREATE TABLE` / `CREATE INDEX` / `COMMENT ON`;其余语句进 `unsupported`/忽略。
|
|
105
|
+
* 全角字符长度、`CASE_SENSITIVE` 语义等**库侧语义**不在解析层处理,交由规则层。
|
|
106
|
+
* 单条语句内部的极端嵌套(Oracle `q'[...]'` 引号、`$$` 块)未覆盖。
|
|
107
|
+
|
|
108
|
+
## 开发
|
|
109
|
+
|
|
110
|
+
```bash
|
|
111
|
+
uv sync
|
|
112
|
+
uv run pytest -q
|
|
113
|
+
uv run ruff check src tests
|
|
114
|
+
uv run mypy
|
|
115
|
+
```
|
|
116
|
+
|
|
117
|
+
## 许可
|
|
118
|
+
|
|
119
|
+
MIT
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["hatchling>=1.27"]
|
|
3
|
+
build-backend = "hatchling.build"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "ddlkit"
|
|
7
|
+
version = "0.1.0"
|
|
8
|
+
description = "多方言 DDL 文本解析器:无损保真地提取表、列、约束、索引与方言特性"
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
requires-python = ">=3.10"
|
|
11
|
+
license = "MIT"
|
|
12
|
+
license-files = ["LICENSE"]
|
|
13
|
+
authors = [{ name = "yangyang" }]
|
|
14
|
+
keywords = ["ddl", "sql", "parser", "audit", "governance", "clickhouse", "hive", "oceanbase", "dameng"]
|
|
15
|
+
classifiers = [
|
|
16
|
+
"Development Status :: 3 - Alpha",
|
|
17
|
+
"Intended Audience :: Developers",
|
|
18
|
+
"License :: OSI Approved :: MIT License",
|
|
19
|
+
"Operating System :: OS Independent",
|
|
20
|
+
"Programming Language :: Python :: 3",
|
|
21
|
+
"Programming Language :: Python :: 3.10",
|
|
22
|
+
"Programming Language :: Python :: 3.11",
|
|
23
|
+
"Programming Language :: Python :: 3.12",
|
|
24
|
+
"Programming Language :: Python :: 3.13",
|
|
25
|
+
"Topic :: Database",
|
|
26
|
+
"Topic :: Software Development :: Quality Assurance",
|
|
27
|
+
"Typing :: Typed",
|
|
28
|
+
]
|
|
29
|
+
dependencies = [
|
|
30
|
+
# 必须显式声明为纯 Python 版 sqlglot。
|
|
31
|
+
# sqlglot[c](即 sqlglotc 发行包)会用编译产物覆盖 sqlglot/tokenizer_core,
|
|
32
|
+
# 本包依赖纯 Python 的词法实现,未在该形态下验证,因此不要引入 sqlglot[c]。
|
|
33
|
+
"sqlglot>=30.18,<31",
|
|
34
|
+
]
|
|
35
|
+
|
|
36
|
+
[project.urls]
|
|
37
|
+
Homepage = "https://github.com/YOUR-ORG/ddl-kit"
|
|
38
|
+
Repository = "https://github.com/YOUR-ORG/ddl-kit"
|
|
39
|
+
Issues = "https://github.com/YOUR-ORG/ddl-kit/issues"
|
|
40
|
+
Changelog = "https://github.com/YOUR-ORG/ddl-kit/blob/main/CHANGELOG.md"
|
|
41
|
+
|
|
42
|
+
[dependency-groups]
|
|
43
|
+
dev = [
|
|
44
|
+
"pytest>=8.0",
|
|
45
|
+
"pytest-cov>=6.0",
|
|
46
|
+
"ruff>=0.9",
|
|
47
|
+
"mypy>=1.13",
|
|
48
|
+
]
|
|
49
|
+
|
|
50
|
+
[tool.hatch.build.targets.wheel]
|
|
51
|
+
packages = ["src/ddlkit"]
|
|
52
|
+
|
|
53
|
+
[tool.hatch.build.targets.sdist]
|
|
54
|
+
include = ["src/ddlkit", "tests", "README.md", "LICENSE", "CHANGELOG.md"]
|
|
55
|
+
|
|
56
|
+
[tool.pytest.ini_options]
|
|
57
|
+
testpaths = ["tests"]
|
|
58
|
+
addopts = "-q"
|
|
59
|
+
|
|
60
|
+
[tool.ruff]
|
|
61
|
+
line-length = 100
|
|
62
|
+
target-version = "py310"
|
|
63
|
+
|
|
64
|
+
[tool.mypy]
|
|
65
|
+
python_version = "3.10"
|
|
66
|
+
strict = true
|
|
67
|
+
files = ["src/ddlkit"]
|
|
68
|
+
|
|
69
|
+
# 私有索引(内网)示例:把下面这段放进仓库根的 pyproject.toml 或用户级配置,
|
|
70
|
+
# 然后用 `uv publish --index nexus` 发布。
|
|
71
|
+
#
|
|
72
|
+
# [[tool.uv.index]]
|
|
73
|
+
# name = "nexus"
|
|
74
|
+
# url = "https://nexus.internal/repository/pypi-hosted/simple/"
|
|
75
|
+
# publish-url = "https://nexus.internal/repository/pypi-hosted/"
|
|
76
|
+
# explicit = true
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
"""ddlkit —— 多方言 DDL 文本解析器(无损保真)。
|
|
2
|
+
|
|
3
|
+
只做一件事:**DDL 文本 -> 结构化数据**。不含规则引擎。
|
|
4
|
+
"""
|
|
5
|
+
from __future__ import annotations
|
|
6
|
+
|
|
7
|
+
from .api import DIALECT_ALIASES, parse_ddl, parse_file, resolve_dialect
|
|
8
|
+
from .model import Column, Constraint, Index, ParseResult, SourceRef, Table
|
|
9
|
+
|
|
10
|
+
__version__ = "0.1.0"
|
|
11
|
+
|
|
12
|
+
__all__ = [
|
|
13
|
+
"Column",
|
|
14
|
+
"Constraint",
|
|
15
|
+
"DIALECT_ALIASES",
|
|
16
|
+
"Index",
|
|
17
|
+
"ParseResult",
|
|
18
|
+
"SourceRef",
|
|
19
|
+
"Table",
|
|
20
|
+
"__version__",
|
|
21
|
+
"parse_ddl",
|
|
22
|
+
"parse_file",
|
|
23
|
+
"resolve_dialect",
|
|
24
|
+
]
|
|
@@ -0,0 +1,110 @@
|
|
|
1
|
+
"""对外入口。
|
|
2
|
+
|
|
3
|
+
调用方只需要知道两件事:**文本从哪来** 和 **它是哪个方言**。
|
|
4
|
+
包内部负责:编码探测 -> 导出格式识别 -> 语句切分 -> 词法 -> 结构提取 -> 注释回填。
|
|
5
|
+
"""
|
|
6
|
+
from __future__ import annotations
|
|
7
|
+
|
|
8
|
+
from pathlib import Path
|
|
9
|
+
|
|
10
|
+
from . import comments as _comments
|
|
11
|
+
from . import encoding as _encoding
|
|
12
|
+
from . import source as _source
|
|
13
|
+
from .extract import extract_index, extract_table
|
|
14
|
+
from .lexer import lex
|
|
15
|
+
from .model import ParseResult, SourceRef, Table
|
|
16
|
+
|
|
17
|
+
#: 业务库名 -> sqlglot 方言名。新增库只需在这里加一行。
|
|
18
|
+
DIALECT_ALIASES: dict[str, str] = {
|
|
19
|
+
"dm": "oracle",
|
|
20
|
+
"dameng": "oracle",
|
|
21
|
+
"达梦": "oracle",
|
|
22
|
+
"ck": "clickhouse",
|
|
23
|
+
"clickhouse": "clickhouse",
|
|
24
|
+
"ob": "mysql",
|
|
25
|
+
"ob_mysql": "mysql",
|
|
26
|
+
"oceanbase": "mysql",
|
|
27
|
+
"oceanbase_mysql": "mysql",
|
|
28
|
+
"hive": "hive",
|
|
29
|
+
"hive2": "hive",
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def resolve_dialect(name: str) -> str:
|
|
34
|
+
"""把业务库名映射成 sqlglot 方言名;已是方言名则原样返回。"""
|
|
35
|
+
key = name.strip().lower()
|
|
36
|
+
return DIALECT_ALIASES.get(key, key)
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def parse_ddl(
|
|
40
|
+
text: str | bytes,
|
|
41
|
+
dialect: str,
|
|
42
|
+
*,
|
|
43
|
+
filename: str | None = None,
|
|
44
|
+
encoding: str | None = None,
|
|
45
|
+
fmt: str | None = None,
|
|
46
|
+
) -> ParseResult:
|
|
47
|
+
"""解析一段 DDL 文本(可含多库/多语句)。"""
|
|
48
|
+
if isinstance(text, bytes):
|
|
49
|
+
text, used_encoding = _encoding.decode(text, encoding)
|
|
50
|
+
else:
|
|
51
|
+
used_encoding = encoding or "utf-8"
|
|
52
|
+
|
|
53
|
+
resolved = resolve_dialect(dialect)
|
|
54
|
+
fmt = fmt or _source.detect(text)
|
|
55
|
+
|
|
56
|
+
result = ParseResult()
|
|
57
|
+
statements = _source.split(text, fmt)
|
|
58
|
+
|
|
59
|
+
tables: list[Table] = []
|
|
60
|
+
by_name: dict[str, Table] = {}
|
|
61
|
+
pending_indexes: list[tuple[str, object, int]] = []
|
|
62
|
+
comment_on_texts: list[str] = []
|
|
63
|
+
|
|
64
|
+
for stmt in statements:
|
|
65
|
+
if stmt.kind == "other":
|
|
66
|
+
continue
|
|
67
|
+
lexed = lex(stmt.sql, resolved)
|
|
68
|
+
src = SourceRef(path=filename, line=stmt.line, encoding=used_encoding, fmt=fmt)
|
|
69
|
+
|
|
70
|
+
if stmt.kind == "create_table":
|
|
71
|
+
table = extract_table(lexed, resolved, src)
|
|
72
|
+
if table is None:
|
|
73
|
+
result.unsupported.append(f"line {stmt.line}: {stmt.sql[:80]}")
|
|
74
|
+
continue
|
|
75
|
+
tables.append(table)
|
|
76
|
+
if table.schema:
|
|
77
|
+
by_name[f"{table.schema}.{table.name}".upper()] = table
|
|
78
|
+
by_name.setdefault(table.name.upper(), table)
|
|
79
|
+
elif stmt.kind == "create_index":
|
|
80
|
+
parsed = extract_index(lexed, src)
|
|
81
|
+
if parsed is not None:
|
|
82
|
+
pending_indexes.append((parsed[0], parsed[1], stmt.line))
|
|
83
|
+
elif stmt.kind == "comment_on":
|
|
84
|
+
comment_on_texts.append(stmt.sql)
|
|
85
|
+
|
|
86
|
+
for target, index, line in pending_indexes:
|
|
87
|
+
table = by_name.get(target.upper())
|
|
88
|
+
if table is None:
|
|
89
|
+
result.warnings.append(f"line {line}: 索引 {index.name!r} 找不到目标表 {target!r}")
|
|
90
|
+
continue
|
|
91
|
+
table.indexes.append(index)
|
|
92
|
+
|
|
93
|
+
if comment_on_texts:
|
|
94
|
+
_comments.backfill(comment_on_texts, tables)
|
|
95
|
+
|
|
96
|
+
result.tables = tables
|
|
97
|
+
return result
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
def parse_file(
|
|
101
|
+
path: str | Path,
|
|
102
|
+
dialect: str,
|
|
103
|
+
*,
|
|
104
|
+
encoding: str | None = None,
|
|
105
|
+
fmt: str | None = None,
|
|
106
|
+
) -> ParseResult:
|
|
107
|
+
"""解析一个 DDL 导出文件。"""
|
|
108
|
+
p = Path(path)
|
|
109
|
+
text, used = _encoding.decode(p.read_bytes(), encoding)
|
|
110
|
+
return parse_ddl(text, dialect, filename=p.name, encoding=used, fmt=fmt)
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
"""``COMMENT ON`` 语句回填。
|
|
2
|
+
|
|
3
|
+
Oracle 系的库(达梦)把注释写成**独立语句**:
|
|
4
|
+
|
|
5
|
+
COMMENT ON TABLE "S"."T" IS '表注释'
|
|
6
|
+
COMMENT ON COLUMN "S"."T"."C" IS '列注释'
|
|
7
|
+
|
|
8
|
+
只解析单条 CREATE TABLE 会漏掉全部注释,导致"注释必填"类规则大面积误报。
|
|
9
|
+
该语法极规整,正则可 100% 覆盖,比赌解析器对 COMMENT 语句的支持更稳。
|
|
10
|
+
"""
|
|
11
|
+
from __future__ import annotations
|
|
12
|
+
|
|
13
|
+
import re
|
|
14
|
+
|
|
15
|
+
from .model import Table
|
|
16
|
+
|
|
17
|
+
COMMENT_ON = re.compile(
|
|
18
|
+
r"""COMMENT\s+ON\s+(?P<kind>TABLE|COLUMN)\s+
|
|
19
|
+
(?P<target>
|
|
20
|
+
"[^"]+"(?:\s*\.\s*"[^"]+")*
|
|
21
|
+
| `[^`]+`(?:\s*\.\s*`[^`]+`)*
|
|
22
|
+
| [A-Za-z_][\w$#]*(?:\s*\.\s*[A-Za-z_][\w$#]*)*
|
|
23
|
+
)
|
|
24
|
+
\s+IS\s+'(?P<text>(?:[^']|'')*)'""",
|
|
25
|
+
re.IGNORECASE | re.VERBOSE,
|
|
26
|
+
)
|
|
27
|
+
|
|
28
|
+
_NAME_SPLIT = re.compile(r"\s*\.\s*")
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def _parts(target: str) -> list[str]:
|
|
32
|
+
return [p.strip('"`').upper() for p in _NAME_SPLIT.split(target.strip())]
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def _build_index(tables: list[Table]) -> dict[tuple[str, ...], Table]:
|
|
36
|
+
index: dict[tuple[str, ...], Table] = {}
|
|
37
|
+
for t in tables:
|
|
38
|
+
if t.schema:
|
|
39
|
+
index[(t.schema.upper(), t.name.upper())] = t
|
|
40
|
+
index[(t.name.upper(),)] = t
|
|
41
|
+
return index
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def backfill(statements: list[str], tables: list[Table]) -> int:
|
|
45
|
+
"""把 COMMENT ON 回填到表/列上,返回成功回填的条数。"""
|
|
46
|
+
index = _build_index(tables)
|
|
47
|
+
hits = 0
|
|
48
|
+
for stmt in statements:
|
|
49
|
+
for m in COMMENT_ON.finditer(stmt):
|
|
50
|
+
kind = m.group("kind").upper()
|
|
51
|
+
segs = _parts(m.group("target"))
|
|
52
|
+
text = m.group("text").replace("''", "'")
|
|
53
|
+
if not segs:
|
|
54
|
+
continue
|
|
55
|
+
|
|
56
|
+
if kind == "TABLE":
|
|
57
|
+
table = index.get(tuple(segs[-2:])) or index.get((segs[-1],))
|
|
58
|
+
if table is not None:
|
|
59
|
+
table.comment = text
|
|
60
|
+
hits += 1
|
|
61
|
+
else:
|
|
62
|
+
if len(segs) < 2:
|
|
63
|
+
continue
|
|
64
|
+
table = index.get(tuple(segs[-3:-1])) or index.get((segs[-2],))
|
|
65
|
+
if table is None:
|
|
66
|
+
continue
|
|
67
|
+
for col in table.columns:
|
|
68
|
+
if col.name.upper() == segs[-1]:
|
|
69
|
+
col.comment = text
|
|
70
|
+
hits += 1
|
|
71
|
+
break
|
|
72
|
+
return hits
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
"""编码探测。
|
|
2
|
+
|
|
3
|
+
实测:同一批导出文件里 UTF-8 与 GB18030 混存(OB 的 test.sql 是 GB18030,
|
|
4
|
+
其余 88 个文件是 UTF-8)。按单一编码读取会让中文注释整体乱码,
|
|
5
|
+
而"注释必填"类规则会因此大面积误报。
|
|
6
|
+
"""
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
BOM_UTF8 = b"\xef\xbb\xbf"
|
|
10
|
+
CANDIDATES: tuple[str, ...] = ("utf-8", "gb18030")
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def sniff(raw: bytes) -> str:
|
|
14
|
+
"""返回最可能的编码名。
|
|
15
|
+
|
|
16
|
+
顺序敏感:UTF-8 先试,失败再试 GB18030。
|
|
17
|
+
局限:字节序列恰好同时合法于两种编码时会误判(中文文本极少见)。
|
|
18
|
+
需要更高准确率时可换 charset-normalizer,本包默认零依赖。
|
|
19
|
+
"""
|
|
20
|
+
if raw.startswith(BOM_UTF8):
|
|
21
|
+
return "utf-8-sig"
|
|
22
|
+
for enc in CANDIDATES:
|
|
23
|
+
try:
|
|
24
|
+
raw.decode(enc)
|
|
25
|
+
return enc
|
|
26
|
+
except UnicodeDecodeError:
|
|
27
|
+
continue
|
|
28
|
+
return "utf-8" # 兜底,配合 errors="replace"
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def decode(raw: bytes, encoding: str | None = None) -> tuple[str, str]:
|
|
32
|
+
"""返回 (文本, 实际使用的编码名)。"""
|
|
33
|
+
enc = encoding or sniff(raw)
|
|
34
|
+
return raw.decode(enc, errors="strict" if encoding else "replace"), enc
|