sdpy-kit 0.2.3__tar.gz → 0.2.5__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- sdpy_kit-0.2.3/README.md → sdpy_kit-0.2.5/PKG-INFO +124 -8
- sdpy_kit-0.2.3/PKG-INFO → sdpy_kit-0.2.5/README.md +54 -78
- {sdpy_kit-0.2.3 → sdpy_kit-0.2.5}/kit/__init__.py +4 -1
- {sdpy_kit-0.2.3 → sdpy_kit-0.2.5}/kit/db/__init__.py +4 -0
- {sdpy_kit-0.2.3 → sdpy_kit-0.2.5}/kit/db/base_db_kit.py +302 -25
- {sdpy_kit-0.2.3 → sdpy_kit-0.2.5}/kit/db/mysql_kit.py +31 -63
- {sdpy_kit-0.2.3 → sdpy_kit-0.2.5}/kit/db/pg_kit.py +27 -55
- {sdpy_kit-0.2.3 → sdpy_kit-0.2.5}/kit/db/sqlite_kit.py +33 -55
- {sdpy_kit-0.2.3 → sdpy_kit-0.2.5}/pyproject.toml +10 -10
- {sdpy_kit-0.2.3 → sdpy_kit-0.2.5}/.gitignore +0 -0
- {sdpy_kit-0.2.3 → sdpy_kit-0.2.5}/kit/ai/__init__.py +0 -0
- {sdpy_kit-0.2.3 → sdpy_kit-0.2.5}/kit/ai/base_service.py +0 -0
- {sdpy_kit-0.2.3 → sdpy_kit-0.2.5}/kit/ai/chat_service.py +0 -0
- {sdpy_kit-0.2.3 → sdpy_kit-0.2.5}/kit/ai/embedding_service.py +0 -0
- {sdpy_kit-0.2.3 → sdpy_kit-0.2.5}/kit/ai/exceptions.py +0 -0
- {sdpy_kit-0.2.3 → sdpy_kit-0.2.5}/kit/ai/failover.py +0 -0
- {sdpy_kit-0.2.3 → sdpy_kit-0.2.5}/kit/ai/image_service.py +0 -0
- {sdpy_kit-0.2.3 → sdpy_kit-0.2.5}/kit/ai/model_builder.py +0 -0
- {sdpy_kit-0.2.3 → sdpy_kit-0.2.5}/kit/ai/model_client.py +0 -0
- {sdpy_kit-0.2.3 → sdpy_kit-0.2.5}/kit/ai/pool.py +0 -0
- {sdpy_kit-0.2.3 → sdpy_kit-0.2.5}/kit/ai/profiles.py +0 -0
- {sdpy_kit-0.2.3 → sdpy_kit-0.2.5}/kit/ai/rerank_service.py +0 -0
- {sdpy_kit-0.2.3 → sdpy_kit-0.2.5}/kit/ai/runner.py +0 -0
- {sdpy_kit-0.2.3 → sdpy_kit-0.2.5}/kit/ai/types.py +0 -0
- {sdpy_kit-0.2.3 → sdpy_kit-0.2.5}/kit/anydoc_kit.py +0 -0
- {sdpy_kit-0.2.3 → sdpy_kit-0.2.5}/kit/common_kit.py +0 -0
- {sdpy_kit-0.2.3 → sdpy_kit-0.2.5}/kit/config/__init__.py +0 -0
- {sdpy_kit-0.2.3 → sdpy_kit-0.2.5}/kit/config/kit_config.py +0 -0
- {sdpy_kit-0.2.3 → sdpy_kit-0.2.5}/kit/config/log_config.py +0 -0
- {sdpy_kit-0.2.3 → sdpy_kit-0.2.5}/kit/db/crud.py +0 -0
- {sdpy_kit-0.2.3 → sdpy_kit-0.2.5}/kit/db/engine.py +0 -0
- {sdpy_kit-0.2.3 → sdpy_kit-0.2.5}/kit/db/exceptions.py +0 -0
- {sdpy_kit-0.2.3 → sdpy_kit-0.2.5}/kit/db/models/__init__.py +0 -0
- {sdpy_kit-0.2.3 → sdpy_kit-0.2.5}/kit/db/models/base.py +0 -0
- {sdpy_kit-0.2.3 → sdpy_kit-0.2.5}/kit/db/query.py +0 -0
- {sdpy_kit-0.2.3 → sdpy_kit-0.2.5}/kit/db/session.py +0 -0
- {sdpy_kit-0.2.3 → sdpy_kit-0.2.5}/kit/db/transaction.py +0 -0
- {sdpy_kit-0.2.3 → sdpy_kit-0.2.5}/kit/docx_kit.py +0 -0
- {sdpy_kit-0.2.3 → sdpy_kit-0.2.5}/kit/file_kit.py +0 -0
- {sdpy_kit-0.2.3 → sdpy_kit-0.2.5}/kit/marker_kit.py +0 -0
- {sdpy_kit-0.2.3 → sdpy_kit-0.2.5}/kit/mcp_kit.py +0 -0
- {sdpy_kit-0.2.3 → sdpy_kit-0.2.5}/kit/md2doc_kit.py +0 -0
- {sdpy_kit-0.2.3 → sdpy_kit-0.2.5}/kit/mineru_kit.py +0 -0
- {sdpy_kit-0.2.3 → sdpy_kit-0.2.5}/kit/redis_kit.py +0 -0
- {sdpy_kit-0.2.3 → sdpy_kit-0.2.5}/kit/wx_article_kit.py +0 -0
- {sdpy_kit-0.2.3 → sdpy_kit-0.2.5}/kit/zvec_kit.py +0 -0
|
@@ -1,3 +1,73 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: sdpy-kit
|
|
3
|
+
Version: 0.2.5
|
|
4
|
+
Summary: SDPY Kit——统一数据库访问层(PG/MySQL/SQLite)与 AI/MCP/Redis/文件/文档/向量工具集
|
|
5
|
+
Author-email: Clark <changhongyuan@126.com>
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
Keywords: ai,kit,mcp,mysql,postgresql,redis,sdpy,sqlalchemy,sqlite,sqlmodel,向量检索,工具集,开发工具,数据库
|
|
8
|
+
Classifier: Development Status :: 4 - Beta
|
|
9
|
+
Classifier: Intended Audience :: Developers
|
|
10
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
11
|
+
Classifier: Natural Language :: Chinese (Simplified)
|
|
12
|
+
Classifier: Operating System :: OS Independent
|
|
13
|
+
Classifier: Programming Language :: Python :: 3 :: Only
|
|
14
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
16
|
+
Classifier: Topic :: Database
|
|
17
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
18
|
+
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
19
|
+
Requires-Python: >=3.13
|
|
20
|
+
Requires-Dist: pydantic-settings>=2.15.0
|
|
21
|
+
Requires-Dist: pydantic>=2.14.1
|
|
22
|
+
Requires-Dist: sqlalchemy~=2.0.54
|
|
23
|
+
Requires-Dist: sqlmodel>=0.0.48
|
|
24
|
+
Provides-Extra: ai
|
|
25
|
+
Requires-Dist: httpx>=0.28.1; extra == 'ai'
|
|
26
|
+
Requires-Dist: python-dotenv>=1.2.4; extra == 'ai'
|
|
27
|
+
Provides-Extra: all
|
|
28
|
+
Requires-Dist: docxtpl>=0.20.2; extra == 'all'
|
|
29
|
+
Requires-Dist: fastmcp>=4.1.0; extra == 'all'
|
|
30
|
+
Requires-Dist: firecrawl-anydoc>=0.2.4; extra == 'all'
|
|
31
|
+
Requires-Dist: httpx>=0.28.1; extra == 'all'
|
|
32
|
+
Requires-Dist: markdown-it-py>=4.2.0; extra == 'all'
|
|
33
|
+
Requires-Dist: marker-pdf[full]>=2.0.0; extra == 'all'
|
|
34
|
+
Requires-Dist: numpy>=2.5.4; extra == 'all'
|
|
35
|
+
Requires-Dist: pikepdf>=10.17.0; extra == 'all'
|
|
36
|
+
Requires-Dist: pimd[pdfa]>=2.2.5; extra == 'all'
|
|
37
|
+
Requires-Dist: psycopg2-binary>=2.9.13; extra == 'all'
|
|
38
|
+
Requires-Dist: pymysql>=1.2.3; extra == 'all'
|
|
39
|
+
Requires-Dist: python-dotenv>=1.2.4; extra == 'all'
|
|
40
|
+
Requires-Dist: redis>=8.1.0; extra == 'all'
|
|
41
|
+
Requires-Dist: requests>=2.34.2; extra == 'all'
|
|
42
|
+
Requires-Dist: sqliteai-vector>=1.1.2; extra == 'all'
|
|
43
|
+
Requires-Dist: zvec>=0.7.0; extra == 'all'
|
|
44
|
+
Provides-Extra: anydoc
|
|
45
|
+
Requires-Dist: firecrawl-anydoc>=0.2.4; extra == 'anydoc'
|
|
46
|
+
Requires-Dist: pikepdf>=10.17.0; extra == 'anydoc'
|
|
47
|
+
Provides-Extra: docx
|
|
48
|
+
Requires-Dist: docxtpl>=0.20.2; extra == 'docx'
|
|
49
|
+
Provides-Extra: marker
|
|
50
|
+
Requires-Dist: marker-pdf[full]>=2.0.0; extra == 'marker'
|
|
51
|
+
Provides-Extra: mcp
|
|
52
|
+
Requires-Dist: fastmcp>=4.1.0; extra == 'mcp'
|
|
53
|
+
Provides-Extra: md2doc
|
|
54
|
+
Requires-Dist: markdown-it-py>=4.2.0; extra == 'md2doc'
|
|
55
|
+
Requires-Dist: pimd[pdfa]>=2.2.5; extra == 'md2doc'
|
|
56
|
+
Provides-Extra: mineru
|
|
57
|
+
Requires-Dist: requests>=2.34.2; extra == 'mineru'
|
|
58
|
+
Provides-Extra: mysql
|
|
59
|
+
Requires-Dist: pymysql>=1.2.3; extra == 'mysql'
|
|
60
|
+
Provides-Extra: pg
|
|
61
|
+
Requires-Dist: psycopg2-binary>=2.9.13; extra == 'pg'
|
|
62
|
+
Provides-Extra: redis
|
|
63
|
+
Requires-Dist: redis>=8.1.0; extra == 'redis'
|
|
64
|
+
Provides-Extra: sqliteai
|
|
65
|
+
Requires-Dist: numpy>=2.5.4; extra == 'sqliteai'
|
|
66
|
+
Requires-Dist: sqliteai-vector>=1.1.2; extra == 'sqliteai'
|
|
67
|
+
Provides-Extra: zvec
|
|
68
|
+
Requires-Dist: zvec>=0.7.0; extra == 'zvec'
|
|
69
|
+
Description-Content-Type: text/markdown
|
|
70
|
+
|
|
1
71
|
# sdpy-kit
|
|
2
72
|
|
|
3
73
|
SDPY 统一工具集(Kit 体系),供 Python 项目开箱即用:
|
|
@@ -266,7 +336,7 @@ from kit.db import session_scope, transactional, DBError # 推荐
|
|
|
266
336
|
from kit import session_scope, transactional, DBError # 等价
|
|
267
337
|
```
|
|
268
338
|
|
|
269
|
-
### 1. 三个数据库 Kit(
|
|
339
|
+
### 1. 三个数据库 Kit(24 个共享方法 API 完全一致)
|
|
270
340
|
|
|
271
341
|
| 类 | 导入 | 数据库 | extra |
|
|
272
342
|
|----|------|--------|-------|
|
|
@@ -292,7 +362,8 @@ from kit import session_scope, transactional, DBError # 等价
|
|
|
292
362
|
| ORM | `upsert(model, data, conflict_fields) -> Model` | 原子插入或更新(`conflict_fields` 须为唯一约束) |
|
|
293
363
|
| ORM | `count(model) -> int` | 记录总数 |
|
|
294
364
|
| 原生 SQL | `query(sql, params=None) -> list[dict]` | 原生查询,返回字典列表 |
|
|
295
|
-
| 原生 SQL | `execute(sql, params=None) -> int` |
|
|
365
|
+
| 原生 SQL | `execute(sql, params=None) -> int` | 原生写入,返回影响行数;`params` 传**字典列表**时批量执行(executemany,行键集须一致、行值须为标量、空列表返回 0) |
|
|
366
|
+
| 原生 SQL | `prepare_statement(sql, params=None) -> (语句, dict \| None)` | **模块级函数**(`from kit.db import prepare_statement`):构造可在**自有事务**内 `session.execute(stmt, bound)` 的语句,支持 `IN :ids`,调用方无须 import `sqlalchemy` |
|
|
296
367
|
| 原生表 | `create_raw_table(table_name, columns, if_not_exists=True) -> None` | 按「列名 → 列定义」建普通表;⚠ 列定义须按目标库方言书写,不做方言翻译 |
|
|
297
368
|
| 原生表 | `raw_insert(table_name, data) -> int` | 插入一行,返回影响行数 |
|
|
298
369
|
| 原生表 | `raw_get_by(table_name, column, value) -> dict \| None` | 按单列等值取首行 |
|
|
@@ -354,8 +425,9 @@ with SQLiteKit.transaction() as session: # 多操作原子提
|
|
|
354
425
|
|
|
355
426
|
| 方法 | 说明 |
|
|
356
427
|
|------|------|
|
|
357
|
-
| `create_vector_table(table_name, dims, id_col="id", column="embedding", extra_cols=None) -> None` | 建含向量列的表;**`dims` 必填**(SQLite 用 `CHECK` 落实为数据库层约束;PG 硬约束并回读校验已存在表;MySQL
|
|
358
|
-
| `insert_vectors(table_name, vectors, ids=None, extra_data=None, id_col="id", column="embedding") -> int` |
|
|
428
|
+
| `create_vector_table(table_name, dims, id_col="id", column="embedding", extra_cols=None) -> None` | 建含向量列的表;**`dims` 必填**(SQLite 用 `CHECK` 落实为数据库层约束;PG 硬约束并回读校验已存在表;MySQL 为上限语义)。`extra_cols` 支持 `{列名: 类型串}`(推荐,与 `create_raw_table` 同形)与 `[{"name","type"}]` 两种形态 |
|
|
429
|
+
| `insert_vectors(table_name, vectors, ids=None, extra_data=None, id_col="id", column="embedding") -> int` | 批量插入向量(**自成一事务并提交**);`ids=None` 时由 Kit 生成雪花 ID;接受 `list[list[float]]` 或二维 numpy 数组 |
|
|
430
|
+
| `build_insert_vectors(table_name, vectors, ids=None, extra_data=None, id_col="id", column="embedding") -> list[tuple[str, list[dict]]]` | **只拼装、不执行、不管事务**:返回按列集分组的 `(SQL, 参数列表)`,供调用方在**自有事务**内写向量行(见「自有事务内写向量」) |
|
|
359
431
|
| `search_vectors(table_name, query_vector, top_k=10, distance_type="L2", filter_sql=None, exact=True, id_col="id", column="embedding") -> list[dict]` | KNN 检索,按距离升序返回 `{"id", "distance"}`;`exact=True`(默认)禁止近似加速;**MySQL 侧抛 `VectorNotSupportedError`** |
|
|
360
432
|
| `vector_version() -> str` | 向量后端版本;不可用时返回 `"unavailable"`(MySQL 侧为服务端版本号) |
|
|
361
433
|
|
|
@@ -440,6 +512,27 @@ PgKit.search_vectors("my_vec", [0.1] * 384, top_k=5, id_col="vid", column="vec")
|
|
|
440
512
|
> 新增(非破坏性):`VECTOR_CAPABILITIES` 能力声明、`id_col` / `column` 逃生口参数、
|
|
441
513
|
> `insert_vectors` 支持二维 numpy 数组与批量提交。
|
|
442
514
|
|
|
515
|
+
#### 0.2.5 变更(开发中,未发布)
|
|
516
|
+
|
|
517
|
+
| 编号 | 变更 | 影响 |
|
|
518
|
+
|------|------|------|
|
|
519
|
+
| — | 新增**模块级公共函数** `prepare_statement(sql, params=None) -> (语句, 归一化参数)`:供调用方在**自有事务**内执行原生 SQL(含 `IN :ids`),**调用方无须 import `sqlalchemy`**;与 `query` / `execute` 同一实现(单一事实源) | **纯新增**:`query` / `transaction` 的签名与行为**零变更**;共享方法数仍为 **24**(模块级函数不计入) |
|
|
520
|
+
| — | `execute` 的 `params` 支持**字典列表**(executemany,同一语句 + N 行参数一次往返);**空列表**不触库返回 `0` | 向后兼容:单字典 / `None` 形态行为不变;批量要求各行字典**键集一致**、行值为**标量**(与 `IN` 展开互斥),违规在本地抛可操作 `ValueError`(含行号),不触库 |
|
|
521
|
+
| — | 文档:自有事务示例改用 `prepare_statement`(不再需要 `from sqlalchemy import text`);补「批量执行」说明与约束 | 文档 |
|
|
522
|
+
|
|
523
|
+
> ⚠ `:name::type` 这类 PostgreSQL 风格 cast **不被 SQLAlchemy 的 `text()` 识别**(0.2.5 之前亦如此),
|
|
524
|
+
> 请改用 `CAST(:name AS type)`。
|
|
525
|
+
|
|
526
|
+
#### 0.2.4 变更
|
|
527
|
+
|
|
528
|
+
| 编号 | 变更 | 影响 |
|
|
529
|
+
|------|------|------|
|
|
530
|
+
| B4 | **入参自洽性校验**(`extra_data` 的键):保留绑定名 `id` / `vec`、与 `id_col` / `column` 重复、不是合法标识符、`id_col == column` → 抛可操作的 `ValueError` | **唯一行为变更**:此前 SQLite 会**静默写坏数据**(如 `id` 列被覆盖)、PG/MySQL 报误导性错误;正常调用**零影响** |
|
|
531
|
+
| — | 新增共享方法 `build_insert_vectors`(只拼装不执行,由 `BaseDbKit` 收敛 → 共享方法 23 → **24**) | 纯新增:可在**自有事务**内写向量行(见下) |
|
|
532
|
+
| — | `create_vector_table` 的 `extra_cols` 支持 `{列名: 类型串}`(与 `create_raw_table` 同形) | 纯新增:既有 `[{"name","type"}]` 形态保留,两者产出 DDL 逐字一致 |
|
|
533
|
+
| — | 后端差异收口:向量列占位符由 `VECTOR_PLACEHOLDER` 驱动(MySQL = `STRING_TO_VECTOR(:vec)`) | 内部整理:公开签名与行为不变 |
|
|
534
|
+
| — | `SQLiteKit.insert_vectors` 的事务获取统一为 `transaction()`(与另两库同构) | 内部一致:行为等价(自成一事务并提交) |
|
|
535
|
+
|
|
443
536
|
#### 0.2.3 变更
|
|
444
537
|
|
|
445
538
|
| 编号 | 变更 | 影响 |
|
|
@@ -457,18 +550,41 @@ rows = SQLiteKit.query("SELECT id, title FROM items WHERE id IN :ids",
|
|
|
457
550
|
{"ids": [h["id"] for h in hits]}) # IN 不保证顺序,需按 hits 重排
|
|
458
551
|
```
|
|
459
552
|
|
|
460
|
-
|
|
553
|
+
**自有事务内写向量(推荐:`build_insert_vectors` + `transaction()`)**:
|
|
461
554
|
|
|
462
555
|
```python
|
|
463
|
-
|
|
556
|
+
from kit.db import SQLiteKit, prepare_statement # 无须 import sqlalchemy
|
|
557
|
+
|
|
558
|
+
with SQLiteKit.transaction() as session: # 换成 PgKit / MySqlKit 写法相同
|
|
559
|
+
stmt, bound = prepare_statement(
|
|
560
|
+
"INSERT INTO docs_item (id, title) VALUES (:id, :t)", {"id": doc_id, "t": title})
|
|
561
|
+
session.execute(stmt, bound) # 第二个值必须一并传入
|
|
562
|
+
for sql, params in SQLiteKit.build_insert_vectors("docs_doc_vectors", vectors,
|
|
563
|
+
ids=row_ids, extra_data=metas,
|
|
564
|
+
id_col="rowid"):
|
|
565
|
+
session.execute(*prepare_statement(sql, params))
|
|
566
|
+
# 正常退出统一提交;异常自动回滚(多处写入保持同一事务)
|
|
567
|
+
```
|
|
568
|
+
|
|
569
|
+
**SQLite 的 DBAPI 变体**(`raw_connection()` 支持 `:name` 命名参数):
|
|
570
|
+
|
|
571
|
+
```python
|
|
572
|
+
conn = SQLiteKit.raw_connection() # 调用方负责 commit / close
|
|
464
573
|
try:
|
|
465
|
-
|
|
466
|
-
|
|
574
|
+
for sql, params in SQLiteKit.build_insert_vectors("item_vectors", vectors,
|
|
575
|
+
ids=row_ids, extra_data=metas):
|
|
576
|
+
conn.executemany(sql, params)
|
|
467
577
|
conn.commit()
|
|
468
578
|
finally:
|
|
469
579
|
conn.close()
|
|
470
580
|
```
|
|
471
581
|
|
|
582
|
+
> ⚠ 两点约束:① `build_insert_vectors` 输出为 `:name` 风格,**PG / MySQL 的原生 DBAPI 不支持**
|
|
583
|
+
> (psycopg2 / pymysql 需 `%(name)s`),只能经 `transaction()` 的 session 执行;
|
|
584
|
+
> ② SQLite 上**持有未提交写事务**时再调 kit 的**自提交**方法(如 `insert_vectors`)会因另一个连接
|
|
585
|
+
> 拿不到写锁而失败(`OperationalError: database is locked`,约 **6.4s** 后才报错)——
|
|
586
|
+
> 把写入并入**同一个** `transaction()`(如上)即可避免。
|
|
587
|
+
|
|
472
588
|
> 相似度换算建议:SQLite 仅支持 L2,若向量入库前已**归一化**,则 `cos = 1 − L2²/2`;
|
|
473
589
|
> PG 可直接用 `distance_type="COSINE"`。
|
|
474
590
|
|
|
@@ -1,73 +1,3 @@
|
|
|
1
|
-
Metadata-Version: 2.5
|
|
2
|
-
Name: sdpy-kit
|
|
3
|
-
Version: 0.2.3
|
|
4
|
-
Summary: SDPY Kit——统一数据库访问层(PG/MySQL/SQLite)与 AI/MCP/Redis/文件/文档/向量工具集
|
|
5
|
-
Author-email: Clark <changhongyuan@126.com>
|
|
6
|
-
License-Expression: MIT
|
|
7
|
-
Keywords: ai,kit,mcp,mysql,postgresql,redis,sdpy,sqlalchemy,sqlite,sqlmodel,向量检索,工具集,开发工具,数据库
|
|
8
|
-
Classifier: Development Status :: 4 - Beta
|
|
9
|
-
Classifier: Intended Audience :: Developers
|
|
10
|
-
Classifier: License :: OSI Approved :: MIT License
|
|
11
|
-
Classifier: Natural Language :: Chinese (Simplified)
|
|
12
|
-
Classifier: Operating System :: OS Independent
|
|
13
|
-
Classifier: Programming Language :: Python :: 3 :: Only
|
|
14
|
-
Classifier: Programming Language :: Python :: 3.13
|
|
15
|
-
Classifier: Programming Language :: Python :: 3.14
|
|
16
|
-
Classifier: Topic :: Database
|
|
17
|
-
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
18
|
-
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
19
|
-
Requires-Python: >=3.13
|
|
20
|
-
Requires-Dist: pydantic-settings>=2.15.0
|
|
21
|
-
Requires-Dist: pydantic>=2.13.5
|
|
22
|
-
Requires-Dist: sqlalchemy~=2.0.54
|
|
23
|
-
Requires-Dist: sqlmodel>=0.0.46
|
|
24
|
-
Provides-Extra: ai
|
|
25
|
-
Requires-Dist: httpx>=0.28.1; extra == 'ai'
|
|
26
|
-
Requires-Dist: python-dotenv>=1.2.3; extra == 'ai'
|
|
27
|
-
Provides-Extra: all
|
|
28
|
-
Requires-Dist: docxtpl>=0.20.2; extra == 'all'
|
|
29
|
-
Requires-Dist: fastmcp>=4.0.5; extra == 'all'
|
|
30
|
-
Requires-Dist: firecrawl-anydoc>=0.2.4; extra == 'all'
|
|
31
|
-
Requires-Dist: httpx>=0.28.1; extra == 'all'
|
|
32
|
-
Requires-Dist: markdown-it-py>=4.2.0; extra == 'all'
|
|
33
|
-
Requires-Dist: marker-pdf[full]>=2.0.0; extra == 'all'
|
|
34
|
-
Requires-Dist: numpy>=2.5.3; extra == 'all'
|
|
35
|
-
Requires-Dist: pikepdf>=10.13.0; extra == 'all'
|
|
36
|
-
Requires-Dist: pimd[pdfa]>=2.2.5; extra == 'all'
|
|
37
|
-
Requires-Dist: psycopg2-binary>=2.9.13; extra == 'all'
|
|
38
|
-
Requires-Dist: pymysql>=1.2.3; extra == 'all'
|
|
39
|
-
Requires-Dist: python-dotenv>=1.2.3; extra == 'all'
|
|
40
|
-
Requires-Dist: redis>=8.1.0; extra == 'all'
|
|
41
|
-
Requires-Dist: requests>=2.34.2; extra == 'all'
|
|
42
|
-
Requires-Dist: sqliteai-vector>=1.1.2; extra == 'all'
|
|
43
|
-
Requires-Dist: zvec>=0.7.0; extra == 'all'
|
|
44
|
-
Provides-Extra: anydoc
|
|
45
|
-
Requires-Dist: firecrawl-anydoc>=0.2.4; extra == 'anydoc'
|
|
46
|
-
Requires-Dist: pikepdf>=10.13.0; extra == 'anydoc'
|
|
47
|
-
Provides-Extra: docx
|
|
48
|
-
Requires-Dist: docxtpl>=0.20.2; extra == 'docx'
|
|
49
|
-
Provides-Extra: marker
|
|
50
|
-
Requires-Dist: marker-pdf[full]>=2.0.0; extra == 'marker'
|
|
51
|
-
Provides-Extra: mcp
|
|
52
|
-
Requires-Dist: fastmcp>=4.0.5; extra == 'mcp'
|
|
53
|
-
Provides-Extra: md2doc
|
|
54
|
-
Requires-Dist: markdown-it-py>=4.2.0; extra == 'md2doc'
|
|
55
|
-
Requires-Dist: pimd[pdfa]>=2.2.5; extra == 'md2doc'
|
|
56
|
-
Provides-Extra: mineru
|
|
57
|
-
Requires-Dist: requests>=2.34.2; extra == 'mineru'
|
|
58
|
-
Provides-Extra: mysql
|
|
59
|
-
Requires-Dist: pymysql>=1.2.3; extra == 'mysql'
|
|
60
|
-
Provides-Extra: pg
|
|
61
|
-
Requires-Dist: psycopg2-binary>=2.9.13; extra == 'pg'
|
|
62
|
-
Provides-Extra: redis
|
|
63
|
-
Requires-Dist: redis>=8.1.0; extra == 'redis'
|
|
64
|
-
Provides-Extra: sqliteai
|
|
65
|
-
Requires-Dist: numpy>=2.5.3; extra == 'sqliteai'
|
|
66
|
-
Requires-Dist: sqliteai-vector>=1.1.2; extra == 'sqliteai'
|
|
67
|
-
Provides-Extra: zvec
|
|
68
|
-
Requires-Dist: zvec>=0.7.0; extra == 'zvec'
|
|
69
|
-
Description-Content-Type: text/markdown
|
|
70
|
-
|
|
71
1
|
# sdpy-kit
|
|
72
2
|
|
|
73
3
|
SDPY 统一工具集(Kit 体系),供 Python 项目开箱即用:
|
|
@@ -336,7 +266,7 @@ from kit.db import session_scope, transactional, DBError # 推荐
|
|
|
336
266
|
from kit import session_scope, transactional, DBError # 等价
|
|
337
267
|
```
|
|
338
268
|
|
|
339
|
-
### 1. 三个数据库 Kit(
|
|
269
|
+
### 1. 三个数据库 Kit(24 个共享方法 API 完全一致)
|
|
340
270
|
|
|
341
271
|
| 类 | 导入 | 数据库 | extra |
|
|
342
272
|
|----|------|--------|-------|
|
|
@@ -362,7 +292,8 @@ from kit import session_scope, transactional, DBError # 等价
|
|
|
362
292
|
| ORM | `upsert(model, data, conflict_fields) -> Model` | 原子插入或更新(`conflict_fields` 须为唯一约束) |
|
|
363
293
|
| ORM | `count(model) -> int` | 记录总数 |
|
|
364
294
|
| 原生 SQL | `query(sql, params=None) -> list[dict]` | 原生查询,返回字典列表 |
|
|
365
|
-
| 原生 SQL | `execute(sql, params=None) -> int` |
|
|
295
|
+
| 原生 SQL | `execute(sql, params=None) -> int` | 原生写入,返回影响行数;`params` 传**字典列表**时批量执行(executemany,行键集须一致、行值须为标量、空列表返回 0) |
|
|
296
|
+
| 原生 SQL | `prepare_statement(sql, params=None) -> (语句, dict \| None)` | **模块级函数**(`from kit.db import prepare_statement`):构造可在**自有事务**内 `session.execute(stmt, bound)` 的语句,支持 `IN :ids`,调用方无须 import `sqlalchemy` |
|
|
366
297
|
| 原生表 | `create_raw_table(table_name, columns, if_not_exists=True) -> None` | 按「列名 → 列定义」建普通表;⚠ 列定义须按目标库方言书写,不做方言翻译 |
|
|
367
298
|
| 原生表 | `raw_insert(table_name, data) -> int` | 插入一行,返回影响行数 |
|
|
368
299
|
| 原生表 | `raw_get_by(table_name, column, value) -> dict \| None` | 按单列等值取首行 |
|
|
@@ -424,8 +355,9 @@ with SQLiteKit.transaction() as session: # 多操作原子提
|
|
|
424
355
|
|
|
425
356
|
| 方法 | 说明 |
|
|
426
357
|
|------|------|
|
|
427
|
-
| `create_vector_table(table_name, dims, id_col="id", column="embedding", extra_cols=None) -> None` | 建含向量列的表;**`dims` 必填**(SQLite 用 `CHECK` 落实为数据库层约束;PG 硬约束并回读校验已存在表;MySQL
|
|
428
|
-
| `insert_vectors(table_name, vectors, ids=None, extra_data=None, id_col="id", column="embedding") -> int` |
|
|
358
|
+
| `create_vector_table(table_name, dims, id_col="id", column="embedding", extra_cols=None) -> None` | 建含向量列的表;**`dims` 必填**(SQLite 用 `CHECK` 落实为数据库层约束;PG 硬约束并回读校验已存在表;MySQL 为上限语义)。`extra_cols` 支持 `{列名: 类型串}`(推荐,与 `create_raw_table` 同形)与 `[{"name","type"}]` 两种形态 |
|
|
359
|
+
| `insert_vectors(table_name, vectors, ids=None, extra_data=None, id_col="id", column="embedding") -> int` | 批量插入向量(**自成一事务并提交**);`ids=None` 时由 Kit 生成雪花 ID;接受 `list[list[float]]` 或二维 numpy 数组 |
|
|
360
|
+
| `build_insert_vectors(table_name, vectors, ids=None, extra_data=None, id_col="id", column="embedding") -> list[tuple[str, list[dict]]]` | **只拼装、不执行、不管事务**:返回按列集分组的 `(SQL, 参数列表)`,供调用方在**自有事务**内写向量行(见「自有事务内写向量」) |
|
|
429
361
|
| `search_vectors(table_name, query_vector, top_k=10, distance_type="L2", filter_sql=None, exact=True, id_col="id", column="embedding") -> list[dict]` | KNN 检索,按距离升序返回 `{"id", "distance"}`;`exact=True`(默认)禁止近似加速;**MySQL 侧抛 `VectorNotSupportedError`** |
|
|
430
362
|
| `vector_version() -> str` | 向量后端版本;不可用时返回 `"unavailable"`(MySQL 侧为服务端版本号) |
|
|
431
363
|
|
|
@@ -510,6 +442,27 @@ PgKit.search_vectors("my_vec", [0.1] * 384, top_k=5, id_col="vid", column="vec")
|
|
|
510
442
|
> 新增(非破坏性):`VECTOR_CAPABILITIES` 能力声明、`id_col` / `column` 逃生口参数、
|
|
511
443
|
> `insert_vectors` 支持二维 numpy 数组与批量提交。
|
|
512
444
|
|
|
445
|
+
#### 0.2.5 变更(开发中,未发布)
|
|
446
|
+
|
|
447
|
+
| 编号 | 变更 | 影响 |
|
|
448
|
+
|------|------|------|
|
|
449
|
+
| — | 新增**模块级公共函数** `prepare_statement(sql, params=None) -> (语句, 归一化参数)`:供调用方在**自有事务**内执行原生 SQL(含 `IN :ids`),**调用方无须 import `sqlalchemy`**;与 `query` / `execute` 同一实现(单一事实源) | **纯新增**:`query` / `transaction` 的签名与行为**零变更**;共享方法数仍为 **24**(模块级函数不计入) |
|
|
450
|
+
| — | `execute` 的 `params` 支持**字典列表**(executemany,同一语句 + N 行参数一次往返);**空列表**不触库返回 `0` | 向后兼容:单字典 / `None` 形态行为不变;批量要求各行字典**键集一致**、行值为**标量**(与 `IN` 展开互斥),违规在本地抛可操作 `ValueError`(含行号),不触库 |
|
|
451
|
+
| — | 文档:自有事务示例改用 `prepare_statement`(不再需要 `from sqlalchemy import text`);补「批量执行」说明与约束 | 文档 |
|
|
452
|
+
|
|
453
|
+
> ⚠ `:name::type` 这类 PostgreSQL 风格 cast **不被 SQLAlchemy 的 `text()` 识别**(0.2.5 之前亦如此),
|
|
454
|
+
> 请改用 `CAST(:name AS type)`。
|
|
455
|
+
|
|
456
|
+
#### 0.2.4 变更
|
|
457
|
+
|
|
458
|
+
| 编号 | 变更 | 影响 |
|
|
459
|
+
|------|------|------|
|
|
460
|
+
| B4 | **入参自洽性校验**(`extra_data` 的键):保留绑定名 `id` / `vec`、与 `id_col` / `column` 重复、不是合法标识符、`id_col == column` → 抛可操作的 `ValueError` | **唯一行为变更**:此前 SQLite 会**静默写坏数据**(如 `id` 列被覆盖)、PG/MySQL 报误导性错误;正常调用**零影响** |
|
|
461
|
+
| — | 新增共享方法 `build_insert_vectors`(只拼装不执行,由 `BaseDbKit` 收敛 → 共享方法 23 → **24**) | 纯新增:可在**自有事务**内写向量行(见下) |
|
|
462
|
+
| — | `create_vector_table` 的 `extra_cols` 支持 `{列名: 类型串}`(与 `create_raw_table` 同形) | 纯新增:既有 `[{"name","type"}]` 形态保留,两者产出 DDL 逐字一致 |
|
|
463
|
+
| — | 后端差异收口:向量列占位符由 `VECTOR_PLACEHOLDER` 驱动(MySQL = `STRING_TO_VECTOR(:vec)`) | 内部整理:公开签名与行为不变 |
|
|
464
|
+
| — | `SQLiteKit.insert_vectors` 的事务获取统一为 `transaction()`(与另两库同构) | 内部一致:行为等价(自成一事务并提交) |
|
|
465
|
+
|
|
513
466
|
#### 0.2.3 变更
|
|
514
467
|
|
|
515
468
|
| 编号 | 变更 | 影响 |
|
|
@@ -527,18 +480,41 @@ rows = SQLiteKit.query("SELECT id, title FROM items WHERE id IN :ids",
|
|
|
527
480
|
{"ids": [h["id"] for h in hits]}) # IN 不保证顺序,需按 hits 重排
|
|
528
481
|
```
|
|
529
482
|
|
|
530
|
-
|
|
483
|
+
**自有事务内写向量(推荐:`build_insert_vectors` + `transaction()`)**:
|
|
531
484
|
|
|
532
485
|
```python
|
|
533
|
-
|
|
486
|
+
from kit.db import SQLiteKit, prepare_statement # 无须 import sqlalchemy
|
|
487
|
+
|
|
488
|
+
with SQLiteKit.transaction() as session: # 换成 PgKit / MySqlKit 写法相同
|
|
489
|
+
stmt, bound = prepare_statement(
|
|
490
|
+
"INSERT INTO docs_item (id, title) VALUES (:id, :t)", {"id": doc_id, "t": title})
|
|
491
|
+
session.execute(stmt, bound) # 第二个值必须一并传入
|
|
492
|
+
for sql, params in SQLiteKit.build_insert_vectors("docs_doc_vectors", vectors,
|
|
493
|
+
ids=row_ids, extra_data=metas,
|
|
494
|
+
id_col="rowid"):
|
|
495
|
+
session.execute(*prepare_statement(sql, params))
|
|
496
|
+
# 正常退出统一提交;异常自动回滚(多处写入保持同一事务)
|
|
497
|
+
```
|
|
498
|
+
|
|
499
|
+
**SQLite 的 DBAPI 变体**(`raw_connection()` 支持 `:name` 命名参数):
|
|
500
|
+
|
|
501
|
+
```python
|
|
502
|
+
conn = SQLiteKit.raw_connection() # 调用方负责 commit / close
|
|
534
503
|
try:
|
|
535
|
-
|
|
536
|
-
|
|
504
|
+
for sql, params in SQLiteKit.build_insert_vectors("item_vectors", vectors,
|
|
505
|
+
ids=row_ids, extra_data=metas):
|
|
506
|
+
conn.executemany(sql, params)
|
|
537
507
|
conn.commit()
|
|
538
508
|
finally:
|
|
539
509
|
conn.close()
|
|
540
510
|
```
|
|
541
511
|
|
|
512
|
+
> ⚠ 两点约束:① `build_insert_vectors` 输出为 `:name` 风格,**PG / MySQL 的原生 DBAPI 不支持**
|
|
513
|
+
> (psycopg2 / pymysql 需 `%(name)s`),只能经 `transaction()` 的 session 执行;
|
|
514
|
+
> ② SQLite 上**持有未提交写事务**时再调 kit 的**自提交**方法(如 `insert_vectors`)会因另一个连接
|
|
515
|
+
> 拿不到写锁而失败(`OperationalError: database is locked`,约 **6.4s** 后才报错)——
|
|
516
|
+
> 把写入并入**同一个** `transaction()`(如上)即可避免。
|
|
517
|
+
|
|
542
518
|
> 相似度换算建议:SQLite 仅支持 L2,若向量入库前已**归一化**,则 `cos = 1 − L2²/2`;
|
|
543
519
|
> PG 可直接用 `distance_type="COSINE"`。
|
|
544
520
|
|
|
@@ -23,7 +23,7 @@ Author: Clark Chang
|
|
|
23
23
|
数据库访问通过 kit.db 子包提供(基于 SQLModel)。
|
|
24
24
|
"""
|
|
25
25
|
|
|
26
|
-
__version__ = "0.2.
|
|
26
|
+
__version__ = "0.2.5"
|
|
27
27
|
|
|
28
28
|
from kit.db import (
|
|
29
29
|
MYSQL,
|
|
@@ -47,6 +47,7 @@ from kit.db import (
|
|
|
47
47
|
get_engine,
|
|
48
48
|
get_session,
|
|
49
49
|
list_tables,
|
|
50
|
+
prepare_statement,
|
|
50
51
|
session_scope,
|
|
51
52
|
table_exists,
|
|
52
53
|
to_vector_literal,
|
|
@@ -85,6 +86,8 @@ __all__ = [
|
|
|
85
86
|
"CRUDBase",
|
|
86
87
|
# 事务
|
|
87
88
|
"transactional",
|
|
89
|
+
# 原生 SQL 语句构造(公共)
|
|
90
|
+
"prepare_statement",
|
|
88
91
|
# 向量工具(公共)
|
|
89
92
|
"to_vector_literal",
|
|
90
93
|
"generate_snowflake_ids",
|
|
@@ -32,6 +32,7 @@ Author: Clark Chang
|
|
|
32
32
|
from kit.db.base_db_kit import (
|
|
33
33
|
DEFAULT_DISTANCE_TYPE,
|
|
34
34
|
generate_snowflake_ids,
|
|
35
|
+
prepare_statement,
|
|
35
36
|
to_vector_literal,
|
|
36
37
|
)
|
|
37
38
|
from kit.db.crud import CRUDBase
|
|
@@ -87,6 +88,9 @@ __all__ = [
|
|
|
87
88
|
"CRUDBase",
|
|
88
89
|
# 事务
|
|
89
90
|
"transactional",
|
|
91
|
+
# 原生 SQL 语句构造(公共)
|
|
92
|
+
# 自有事务内执行原生 SQL(含列表参数 IN :ids)时使用,调用方无须 import sqlalchemy
|
|
93
|
+
"prepare_statement",
|
|
90
94
|
# 向量工具(公共)
|
|
91
95
|
"to_vector_literal",
|
|
92
96
|
"generate_snowflake_ids",
|