zleap-sag 0.4.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (114) hide show
  1. zleap_sag-0.4.0/.gitignore +24 -0
  2. zleap_sag-0.4.0/CHANGELOG.md +83 -0
  3. zleap_sag-0.4.0/LICENSE +21 -0
  4. zleap_sag-0.4.0/PKG-INFO +151 -0
  5. zleap_sag-0.4.0/README.md +101 -0
  6. zleap_sag-0.4.0/alembic.ini +149 -0
  7. zleap_sag-0.4.0/migrations/README +1 -0
  8. zleap_sag-0.4.0/migrations/env.py +67 -0
  9. zleap_sag-0.4.0/migrations/script.py.mako +28 -0
  10. zleap_sag-0.4.0/migrations/versions/ffff3272456c_baseline_schema.py +289 -0
  11. zleap_sag-0.4.0/pyproject.toml +123 -0
  12. zleap_sag-0.4.0/src/zleap/sag/__about__.py +3 -0
  13. zleap_sag-0.4.0/src/zleap/sag/__init__.py +19 -0
  14. zleap_sag-0.4.0/src/zleap/sag/_bootstrap.py +118 -0
  15. zleap_sag-0.4.0/src/zleap/sag/config.py +168 -0
  16. zleap_sag-0.4.0/src/zleap/sag/core/adapters/__init__.py +38 -0
  17. zleap_sag-0.4.0/src/zleap/sag/core/adapters/capabilities.py +40 -0
  18. zleap_sag-0.4.0/src/zleap/sag/core/adapters/defaults.py +161 -0
  19. zleap_sag-0.4.0/src/zleap/sag/core/adapters/interfaces.py +77 -0
  20. zleap_sag-0.4.0/src/zleap/sag/core/adapters/registry.py +89 -0
  21. zleap_sag-0.4.0/src/zleap/sag/core/ai/__init__.py +52 -0
  22. zleap_sag-0.4.0/src/zleap/sag/core/ai/base.py +426 -0
  23. zleap_sag-0.4.0/src/zleap/sag/core/ai/embedding.py +204 -0
  24. zleap_sag-0.4.0/src/zleap/sag/core/ai/factory.py +443 -0
  25. zleap_sag-0.4.0/src/zleap/sag/core/ai/litellm_client.py +108 -0
  26. zleap_sag-0.4.0/src/zleap/sag/core/ai/llm.py +333 -0
  27. zleap_sag-0.4.0/src/zleap/sag/core/ai/models.py +81 -0
  28. zleap_sag-0.4.0/src/zleap/sag/core/ai/rerank.py +258 -0
  29. zleap_sag-0.4.0/src/zleap/sag/core/config/__init__.py +7 -0
  30. zleap_sag-0.4.0/src/zleap/sag/core/config/settings.py +271 -0
  31. zleap_sag-0.4.0/src/zleap/sag/core/prompt/__init__.py +17 -0
  32. zleap_sag-0.4.0/src/zleap/sag/core/prompt/manager.py +240 -0
  33. zleap_sag-0.4.0/src/zleap/sag/core/resources.py +86 -0
  34. zleap_sag-0.4.0/src/zleap/sag/core/storage/__init__.py +42 -0
  35. zleap_sag-0.4.0/src/zleap/sag/core/storage/documents/__init__.py +27 -0
  36. zleap_sag-0.4.0/src/zleap/sag/core/storage/documents/entity_vector.py +30 -0
  37. zleap_sag-0.4.0/src/zleap/sag/core/storage/documents/event_entity_vector.py +30 -0
  38. zleap_sag-0.4.0/src/zleap/sag/core/storage/documents/event_vector.py +47 -0
  39. zleap_sag-0.4.0/src/zleap/sag/core/storage/documents/source_chunk.py +47 -0
  40. zleap_sag-0.4.0/src/zleap/sag/core/storage/elasticsearch.py +691 -0
  41. zleap_sag-0.4.0/src/zleap/sag/core/storage/pgvector_store.py +388 -0
  42. zleap_sag-0.4.0/src/zleap/sag/core/storage/repositories/__init__.py +17 -0
  43. zleap_sag-0.4.0/src/zleap/sag/core/storage/repositories/base.py +105 -0
  44. zleap_sag-0.4.0/src/zleap/sag/core/storage/repositories/entity_repository.py +498 -0
  45. zleap_sag-0.4.0/src/zleap/sag/core/storage/repositories/event_entity_repository.py +477 -0
  46. zleap_sag-0.4.0/src/zleap/sag/core/storage/repositories/event_repository.py +626 -0
  47. zleap_sag-0.4.0/src/zleap/sag/core/storage/repositories/source_chunk_repository.py +278 -0
  48. zleap_sag-0.4.0/src/zleap/sag/db/__init__.py +42 -0
  49. zleap_sag-0.4.0/src/zleap/sag/db/base.py +197 -0
  50. zleap_sag-0.4.0/src/zleap/sag/db/models.py +853 -0
  51. zleap_sag-0.4.0/src/zleap/sag/engine.py +351 -0
  52. zleap_sag-0.4.0/src/zleap/sag/exceptions.py +151 -0
  53. zleap_sag-0.4.0/src/zleap/sag/models/__init__.py +25 -0
  54. zleap_sag-0.4.0/src/zleap/sag/models/base.py +42 -0
  55. zleap_sag-0.4.0/src/zleap/sag/models/entity.py +341 -0
  56. zleap_sag-0.4.0/src/zleap/sag/modules/__init__.py +15 -0
  57. zleap_sag-0.4.0/src/zleap/sag/modules/extract/__init__.py +20 -0
  58. zleap_sag-0.4.0/src/zleap/sag/modules/extract/config.py +140 -0
  59. zleap_sag-0.4.0/src/zleap/sag/modules/extract/extractor.py +715 -0
  60. zleap_sag-0.4.0/src/zleap/sag/modules/extract/parser.py +1190 -0
  61. zleap_sag-0.4.0/src/zleap/sag/modules/extract/processor.py +501 -0
  62. zleap_sag-0.4.0/src/zleap/sag/modules/extract/saver.py +804 -0
  63. zleap_sag-0.4.0/src/zleap/sag/modules/load/__init__.py +21 -0
  64. zleap_sag-0.4.0/src/zleap/sag/modules/load/chunking/__init__.py +53 -0
  65. zleap_sag-0.4.0/src/zleap/sag/modules/load/chunking/assembler/__init__.py +11 -0
  66. zleap_sag-0.4.0/src/zleap/sag/modules/load/chunking/assembler/generic.py +158 -0
  67. zleap_sag-0.4.0/src/zleap/sag/modules/load/chunking/base.py +56 -0
  68. zleap_sag-0.4.0/src/zleap/sag/modules/load/chunking/chunker/__init__.py +11 -0
  69. zleap_sag-0.4.0/src/zleap/sag/modules/load/chunking/chunker/base.py +21 -0
  70. zleap_sag-0.4.0/src/zleap/sag/modules/load/chunking/chunker/markdown.py +100 -0
  71. zleap_sag-0.4.0/src/zleap/sag/modules/load/chunking/chunker/text.py +171 -0
  72. zleap_sag-0.4.0/src/zleap/sag/modules/load/chunking/parser/__init__.py +11 -0
  73. zleap_sag-0.4.0/src/zleap/sag/modules/load/chunking/parser/markdown.py +103 -0
  74. zleap_sag-0.4.0/src/zleap/sag/modules/load/chunking/pipeline.py +48 -0
  75. zleap_sag-0.4.0/src/zleap/sag/modules/load/chunking/tokenizer/README.md +12 -0
  76. zleap_sag-0.4.0/src/zleap/sag/modules/load/chunking/tokenizer/__init__.py +11 -0
  77. zleap_sag-0.4.0/src/zleap/sag/modules/load/chunking/tokenizer/assets/tokenizer.json +757480 -0
  78. zleap_sag-0.4.0/src/zleap/sag/modules/load/chunking/tokenizer/estimator.py +85 -0
  79. zleap_sag-0.4.0/src/zleap/sag/modules/load/chunking/types.py +79 -0
  80. zleap_sag-0.4.0/src/zleap/sag/modules/load/config.py +140 -0
  81. zleap_sag-0.4.0/src/zleap/sag/modules/load/loader.py +753 -0
  82. zleap_sag-0.4.0/src/zleap/sag/modules/load/parser.py +325 -0
  83. zleap_sag-0.4.0/src/zleap/sag/modules/load/processor.py +123 -0
  84. zleap_sag-0.4.0/src/zleap/sag/modules/search/README.md +62 -0
  85. zleap_sag-0.4.0/src/zleap/sag/modules/search/__init__.py +47 -0
  86. zleap_sag-0.4.0/src/zleap/sag/modules/search/atomic.py +1109 -0
  87. zleap_sag-0.4.0/src/zleap/sag/modules/search/config.py +510 -0
  88. zleap_sag-0.4.0/src/zleap/sag/modules/search/multi.py +1116 -0
  89. zleap_sag-0.4.0/src/zleap/sag/modules/search/multi_vector.py +1585 -0
  90. zleap_sag-0.4.0/src/zleap/sag/modules/search/searcher.py +326 -0
  91. zleap_sag-0.4.0/src/zleap/sag/modules/search/step5_strategies.py +502 -0
  92. zleap_sag-0.4.0/src/zleap/sag/modules/search/vector.py +181 -0
  93. zleap_sag-0.4.0/src/zleap/sag/prompts/en/extract.yaml +309 -0
  94. zleap_sag-0.4.0/src/zleap/sag/prompts/en/test_extract.yaml +340 -0
  95. zleap_sag-0.4.0/src/zleap/sag/prompts/extract.yaml +360 -0
  96. zleap_sag-0.4.0/src/zleap/sag/prompts/rerank.yaml +15 -0
  97. zleap_sag-0.4.0/src/zleap/sag/py.typed +0 -0
  98. zleap_sag-0.4.0/src/zleap/sag/results.py +53 -0
  99. zleap_sag-0.4.0/src/zleap/sag/utils/__init__.py +73 -0
  100. zleap_sag-0.4.0/src/zleap/sag/utils/batch.py +243 -0
  101. zleap_sag-0.4.0/src/zleap/sag/utils/logger.py +98 -0
  102. zleap_sag-0.4.0/src/zleap/sag/utils/retry.py +148 -0
  103. zleap_sag-0.4.0/src/zleap/sag/utils/text.py +326 -0
  104. zleap_sag-0.4.0/src/zleap/sag/utils/time.py +149 -0
  105. zleap_sag-0.4.0/src/zleap/sag/utils/token_counter.py +264 -0
  106. zleap_sag-0.4.0/tests/integration/.gitkeep +0 -0
  107. zleap_sag-0.4.0/tests/integration/conftest.py +55 -0
  108. zleap_sag-0.4.0/tests/integration/test_pipeline_smoke.py +69 -0
  109. zleap_sag-0.4.0/tests/unit/mysql_schema.snapshot.sql +245 -0
  110. zleap_sag-0.4.0/tests/unit/test_adapters.py +89 -0
  111. zleap_sag-0.4.0/tests/unit/test_chunk.py +61 -0
  112. zleap_sag-0.4.0/tests/unit/test_mysql_schema_snapshot.py +45 -0
  113. zleap_sag-0.4.0/tests/unit/test_resources.py +117 -0
  114. zleap_sag-0.4.0/tests/unit/test_smoke.py +29 -0
@@ -0,0 +1,24 @@
1
+ # Build
2
+ build/
3
+ dist/
4
+ *.egg-info/
5
+ .eggs/
6
+
7
+ # Env
8
+ .venv/
9
+ venv/
10
+
11
+ # Caches
12
+ __pycache__/
13
+ *.py[cod]
14
+ .pytest_cache/
15
+ .mypy_cache/
16
+ .ruff_cache/
17
+ .coverage
18
+ htmlcov/
19
+
20
+ # Secrets
21
+ .env
22
+
23
+ # OS
24
+ .DS_Store
@@ -0,0 +1,83 @@
1
+ # Changelog
2
+
3
+ All notable changes to this project are documented here.
4
+ Format: [Keep a Changelog](https://keepachangelog.com/); versioning: [SemVer](https://semver.org/).
5
+
6
+ ## [0.4.0] - 2026-07-03
7
+
8
+ ### Changed (BREAKING)
9
+ - **导入命名空间迁移**:`zleap_sag` → **`zleap.sag`**(PEP 420 原生命名空间)。
10
+ `from zleap import sag` / `from zleap.sag import DataEngine, EngineConfig, SagError`。
11
+ 分发名仍为 `zleap-sag`;无 `zleap_sag` 兼容 shim(干净切换)。`SAG_` 环境变量前缀不变。
12
+ - **仓库改为 monorepo**(uv workspace):包落 `packages/sag/`,`src/zleap/sag/`;
13
+ 为未来 `zleap-agent`/`zleap-memo` 等**独立、无耦合**的同命名空间包铺底座(各包独立版本/依赖/发布)。
14
+ - 验证:干净 venv `from zleap import sag` 可用、`import zleap.agent` 干净失败;`make verify` 全绿;
15
+ MySQL+ES 集成回归通过(pgvector/PG/OceanBase 逻辑未变)。
16
+
17
+ ## [0.3.0] - 2026-07-02
18
+
19
+ 工程能力升级(对标 cognee):可适配后端骨架、DI 资源容器、Alembic 迁移、启动健康检查、
20
+ 链路灵活化。MySQL+ES 现网路径**行为零回归**,并首次经真机(MySQL+ES+LLM+Embedding)端到端验证。
21
+
22
+ ### Added
23
+ - **向量库可切换(Elasticsearch / pgvector)**:`vector_provider="pgvector"` 让整套引擎跑在**单个
24
+ PostgreSQL**(关系 + 向量,零 ES)。`PgVectorStore` 实现 ES 客户端方法面(repositories 无改动),
25
+ 表按文档自动建(float 列表→`vector(N)`+HNSW cosine 索引),ES filter_query→SQL WHERE,match→ILIKE。
26
+ **真机验证**:全 4 策略(vector/atomic/multi/multi_es)在 PostgreSQL+pgvector 单库跑通。extras:`[postgres]`。
27
+ - **OceanBase 关系后端**(MySQL 协议兼容,`provider="oceanbase"`,复用 aiomysql 驱动/mysql 方言)。
28
+ **活体真机验证**:在 **OceanBase CE v4.4.2.1** 实例上,`ingest→extract→search`(全 4 策略)全链路跑通;
29
+ `create_all`(LONGTEXT/MEDIUMTEXT/VARBINARY 经 with_variant)+ ORM + 查询在 OB MySQL 模式均正常。
30
+ - **关系库可切换(MySQL / PostgreSQL / SQLite)**:`RelationalConfig(provider=...)` 配置驱动、无硬编码;
31
+ schema 经 `with_variant` 可移植(MySQL DDL 逐字节不变,由快照守卫)+ 全局唯一索引名;`db.base` 按方言
32
+ 建引擎(PG `server_settings` 时区 / SQLite 外键 PRAGMA)。**真机验证**:`create_all` 三方言均通过,
33
+ `ingest→extract→search` 在 **MySQL+ES 与 PostgreSQL+ES** 两套后端各自跑通;Alembic 基线跨方言可迁移。
34
+ 向后兼容:`mysql=MySQLConfig(...)` 自动映射为 `relational(provider="mysql")`。extras:`[postgres]`/`[sqlite]`。
35
+ - **后端适配器层**(`core/adapters/`):LLM/Embedding/Rerank/VectorStore/RelationalStore 接口 +
36
+ 能力模型(`Capability`,如 BM25/`supports_lexical`)+ provider 工厂注册表(`create_*`/`register`,
37
+ 对齐 cognee `create_*_engine`)。默认适配器包装现有 OpenAI/ES/SQLAlchemy 客户端,惰性、行为不变。
38
+ - **DI 资源容器**(`core/resources.py`):`ResourceContainer` + `ContextVar` 单一事实来源解析
39
+ (默认容器=现网 MySQL+ES,无裸全局回退);`DataEngine` 构建并持有容器,暴露 `.resources`。
40
+ - **Alembic 迁移**(`migrations/`):首条迁移=当前 schema;已验证 fresh upgrade / 升降级往返 /
41
+ 对现网库 `stamp head` 无损纳管。`SAG_DB_URL` 覆盖连接串。
42
+ - **启动健康检查(fail-fast)**:`DataEngine.start()` 预检关系库/向量库连通性,失败给明确错误并
43
+ 干净回滚(不泄漏 ContextVar、不留半初始化);可 `DataEngine(cfg, health_check=False)` 关闭。
44
+ - **`DataEngine.chunk()`**:只切不存/传字符串的纯 CPU 分块入口(无需 start、无 DB/ES/LLM)。
45
+ - **litellm 多 provider LLM(可选,对齐 cognee)**:`LLMConfig(provider="litellm")` 切换到 litellm
46
+ 客户端(OpenAI/Anthropic/本地等统一入口);默认仍 `openai`(零回归)。需 `pip install zleap-sag[litellm]`。
47
+ 已真机验证:extract/search 走 litellm→302.ai 全链路通过。
48
+ - 集成冒烟测(`tests/integration/`)+ MySQL DDL 零变更快照守卫(`tests/unit/`)。
49
+
50
+ ### Changed
51
+ - `db.base.init_database()` 默认**不再删表**;drop+重建需显式 `allow_drop=True`(仅限开发)。生产用 Alembic。
52
+ - 新增门面/适配器代码纳入严格 lint/type(`core/adapters`、`core/resources`);逐步收窄 vendored 排除范围。
53
+
54
+ ### Fixed
55
+ - **ES 搜索健壮性**:查询尚未创建的索引(如稀疏数据下的 `entity_vectors`)不再抛 404 崩溃,
56
+ 改为优雅返回空结果(`vector_search`/`search` 均修复)。
57
+ - wheel 构建:移除与 hatchling 自动打包冲突的 `prompts/` `force-include`(路径重复)。
58
+ - mypy:`_bootstrap` 向 `PromptManager` 传 `Path` 而非 `str`。
59
+
60
+ ## [Unreleased]
61
+
62
+ ### Added
63
+ - src-layout 包骨架与规范 `pyproject.toml`(hatchling 后端,动态版本)。
64
+ - 公开 API:`DataEngine`(ingest/extract/search + `init_schema` 幂等建表)、注入式 `EngineConfig`
65
+ (含 `from_env`,`SAG_` 前缀标准环境加载)、解耦结果模型、统一异常基类 `SagError`。
66
+ - `py.typed`、Makefile(`make verify`)、CI、pre-commit、测试目录骨架。
67
+
68
+ ### Changed
69
+ - 迁入核心 `core/db/modules/models/utils` + `prompts/`(原 `pipeline/*`,import 前缀机械改名)。
70
+ - `DataEngine` 直接编排 load/extract/search;旧 `pipelineEngine`/`orchestrator` 已内联并删除。
71
+ - `_bootstrap` 适配:`apply_config_to_env`(配置→环境)、`warmup_prompts`、`close_core_resources`。
72
+ - 运行时依赖按核心实际 import 校准(补 `aiohttp`/`numpy`/`elasticsearch-dsl`/`tiktoken`,删 dead `query` extra)。
73
+ - 质量门:vendored 核心按 `extend-exclude` 暂不纳入严格 lint/type,新增门面层严格把关。
74
+ - 异常基类 `pipelineError` → `SagError`。
75
+
76
+ ### Known limitations
77
+ - 单实例 / 进程级配置(全局连接复用);多配置隔离待依赖注入(v0.2)。
78
+
79
+ ### TODO
80
+ - 真机验证(3.11 venv);集成冒烟测(`ingest→extract→search`)。
81
+
82
+ ## [0.1.0] - 未发布
83
+ - 初始骨架。
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 Zleap Team
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,151 @@
1
+ Metadata-Version: 2.4
2
+ Name: zleap-sag
3
+ Version: 0.4.0
4
+ Summary: SAG data engine: ingest, extract, and retrieve over a lightweight event/entity graph
5
+ Project-URL: Homepage, https://github.com/Zleap-AI/zleap
6
+ Project-URL: Issues, https://github.com/Zleap-AI/zleap/issues
7
+ Author-email: Zleap Team <contact@zleap.ai>
8
+ License-Expression: MIT
9
+ License-File: LICENSE
10
+ Keywords: event-extraction,knowledge-graph,multi-hop,rag,retrieval
11
+ Classifier: Development Status :: 4 - Beta
12
+ Classifier: Intended Audience :: Developers
13
+ Classifier: License :: OSI Approved :: MIT License
14
+ Classifier: Programming Language :: Python :: 3.11
15
+ Classifier: Programming Language :: Python :: 3.12
16
+ Classifier: Typing :: Typed
17
+ Requires-Python: >=3.11
18
+ Requires-Dist: aiohttp>=3.9
19
+ Requires-Dist: aiomysql>=0.2
20
+ Requires-Dist: cryptography>=41
21
+ Requires-Dist: elasticsearch-dsl<9,>=8
22
+ Requires-Dist: elasticsearch<9,>=8
23
+ Requires-Dist: json-repair>=0.58
24
+ Requires-Dist: jsonschema>=4
25
+ Requires-Dist: numpy>=1.26
26
+ Requires-Dist: openai>=1.6
27
+ Requires-Dist: pydantic-settings>=2.1
28
+ Requires-Dist: pydantic>=2.5
29
+ Requires-Dist: pyyaml>=6
30
+ Requires-Dist: sqlalchemy[asyncio]>=2.0
31
+ Requires-Dist: tiktoken>=0.5
32
+ Requires-Dist: tokenizers>=0.22
33
+ Provides-Extra: dev
34
+ Requires-Dist: mypy>=1.11; extra == 'dev'
35
+ Requires-Dist: pre-commit>=3.6; extra == 'dev'
36
+ Requires-Dist: pytest-asyncio>=1; extra == 'dev'
37
+ Requires-Dist: pytest-cov>=7; extra == 'dev'
38
+ Requires-Dist: pytest>=8; extra == 'dev'
39
+ Requires-Dist: ruff>=0.6; extra == 'dev'
40
+ Provides-Extra: litellm
41
+ Requires-Dist: litellm>=1.40; extra == 'litellm'
42
+ Provides-Extra: migrations
43
+ Requires-Dist: alembic>=1.17; extra == 'migrations'
44
+ Provides-Extra: postgres
45
+ Requires-Dist: asyncpg>=0.29; extra == 'postgres'
46
+ Requires-Dist: psycopg2-binary>=2.9; extra == 'postgres'
47
+ Provides-Extra: sqlite
48
+ Requires-Dist: aiosqlite>=0.19; extra == 'sqlite'
49
+ Description-Content-Type: text/markdown
50
+
51
+ # zleap-sag
52
+
53
+ SAG 数据引擎:在轻量"事件/实体图"上做 **数据写入 / 抽取 / 检索 / 查询**。
54
+
55
+ > 分发名 `zleap-sag`,import 名 `zleap_sag`。
56
+ > 接入指南见 `docs/接入指南.md`。
57
+
58
+ ## 状态(v0.2.0)
59
+
60
+ ✅ **已真机验证、生产可用(MySQL + Elasticsearch)**。公开入口 `DataEngine` 编排
61
+ load/extract/search;注入式 `config`(支持 `from_env`)、解耦 `results`、统一 `SagError`、
62
+ 幂等 `init_schema` 均就绪。`make verify` 在 3.11 全绿;`ingest→extract→search`(全 4 策略)
63
+ 已对真机 MySQL+ES+LLM+Embedding 端到端跑通(openai 与 litellm 两种 LLM provider 均验证)。
64
+
65
+ 工程能力(对标 cognee):后端**适配器层 + 能力模型 + provider 注册表**、**DI 资源容器**、
66
+ **Alembic 迁移**、**启动健康检查(fail-fast)**、`chunk()` 只切不存、可选 **litellm** 多 provider。
67
+ 详见 `docs/接入指南.md`。
68
+
69
+ > 后端范围(真机验证):**关系库** MySQL / PostgreSQL / SQLite / OceanBase;**向量库** Elasticsearch / pgvector。
70
+ > 全 4 检索策略在 **MySQL+ES**、**PostgreSQL+ES**、**PostgreSQL+pgvector(单库,零 ES)**、
71
+ > **OceanBase(CE v4.4.2.1)+ES** 四套组合均端到端跑通。见 `docs/接入指南.md`。
72
+
73
+ ## 安装与验证(开发)
74
+
75
+ ```bash
76
+ make install # 创建 .venv + 安装开发依赖(可编辑)
77
+ make verify # 全量验证:lint → type → test → build
78
+ make help # 查看全部命令
79
+
80
+ # 解释器需 >= 3.11;如默认 python3 较旧:
81
+ make PYTHON=python3.11 verify
82
+ ```
83
+
84
+ ## 用法
85
+
86
+ ```python
87
+ from zleap_sag import DataEngine, EngineConfig
88
+ from zleap_sag.config import MySQLConfig, ESConfig, LLMConfig, EmbeddingConfig
89
+
90
+ config = EngineConfig(
91
+ mysql=MySQLConfig(user="sag2", password="...", database="sag2"),
92
+ es=ESConfig(hosts=["http://localhost:9200"]),
93
+ llm=LLMConfig(api_key="...", model="qwen3.6-flash"),
94
+ embedding=EmbeddingConfig(model="bge-large-en-v1.5", dimensions=1024),
95
+ )
96
+
97
+ async with DataEngine(config) as engine:
98
+ await engine.init_schema() # 全新库首次建表(幂等,不删表)
99
+ ing = await engine.ingest("doc.md") # 加载单个文档
100
+ await engine.extract() # 抽取事件/实体
101
+ res = await engine.search("Who founded X?", strategy="multi", top_k=10)
102
+ print(len(res.sections))
103
+ ```
104
+
105
+ 也可从环境变量加载配置(`SAG_` 前缀,`__` 表示嵌套):
106
+
107
+ ```bash
108
+ export SAG_MYSQL__USER=sag2 SAG_MYSQL__PASSWORD=... SAG_MYSQL__DATABASE=sag2
109
+ export SAG_ES__HOSTS='["http://localhost:9200"]'
110
+ export SAG_LLM__API_KEY=... SAG_LLM__MODEL=qwen3.6-flash
111
+ export SAG_EMBEDDING__MODEL=bge-large-en-v1.5
112
+ ```
113
+
114
+ ```python
115
+ config = EngineConfig.from_env() # 或 EngineConfig.from_env(env_file=".env")
116
+ ```
117
+
118
+ 统一异常捕获:
119
+
120
+ ```python
121
+ from zleap_sag import SagError # 所有引擎异常的基类
122
+
123
+ try:
124
+ res = await engine.search("...")
125
+ except SagError as e:
126
+ ...
127
+ ```
128
+
129
+ ## ⚠️ 已知限制(v0.2)
130
+
131
+ **多配置隔离(演进中)**:已引入 DI 资源容器(`ResourceContainer` + `ContextVar` 单源解析),
132
+ 但底层部分核心仍复用进程级全局连接(DB/ES)。因此**同一进程内并发多套不同配置**尚未完全隔离,
133
+ 建议一个进程一套配置;完整隔离随核心逐点切换到容器继续推进。
134
+
135
+ - 关系库可切换 **MySQL / PostgreSQL / SQLite / OceanBase**;向量库可切换 **Elasticsearch / pgvector**(单库 PG 可跑完整引擎)。均已真机验证。
136
+ - `ES` 目前取 `hosts` 列表的第一个地址。
137
+ - `ingest` 以**文件路径**为输入(单文档);字符串/只切不存见 `DataEngine.chunk()`。
138
+
139
+ ## 常用命令
140
+
141
+ | 命令 | 作用 |
142
+ |------|------|
143
+ | `make install` | 创建 `.venv` + 安装开发依赖 |
144
+ | `make lint` | ruff 检查 + 格式校验 |
145
+ | `make format` | ruff 自动修复 + 格式化 |
146
+ | `make type` | mypy 类型检查 |
147
+ | `make test` | 单元测试(跳过 integration) |
148
+ | `make test-integration` | 集成测试(需 docker mysql/es) |
149
+ | `make build` | 构建 sdist + wheel 并 `twine check` |
150
+ | `make verify` | lint → type → test → build 全量验证 |
151
+ | `make clean` | 清理构建与缓存产物 |
@@ -0,0 +1,101 @@
1
+ # zleap-sag
2
+
3
+ SAG 数据引擎:在轻量"事件/实体图"上做 **数据写入 / 抽取 / 检索 / 查询**。
4
+
5
+ > 分发名 `zleap-sag`,import 名 `zleap_sag`。
6
+ > 接入指南见 `docs/接入指南.md`。
7
+
8
+ ## 状态(v0.2.0)
9
+
10
+ ✅ **已真机验证、生产可用(MySQL + Elasticsearch)**。公开入口 `DataEngine` 编排
11
+ load/extract/search;注入式 `config`(支持 `from_env`)、解耦 `results`、统一 `SagError`、
12
+ 幂等 `init_schema` 均就绪。`make verify` 在 3.11 全绿;`ingest→extract→search`(全 4 策略)
13
+ 已对真机 MySQL+ES+LLM+Embedding 端到端跑通(openai 与 litellm 两种 LLM provider 均验证)。
14
+
15
+ 工程能力(对标 cognee):后端**适配器层 + 能力模型 + provider 注册表**、**DI 资源容器**、
16
+ **Alembic 迁移**、**启动健康检查(fail-fast)**、`chunk()` 只切不存、可选 **litellm** 多 provider。
17
+ 详见 `docs/接入指南.md`。
18
+
19
+ > 后端范围(真机验证):**关系库** MySQL / PostgreSQL / SQLite / OceanBase;**向量库** Elasticsearch / pgvector。
20
+ > 全 4 检索策略在 **MySQL+ES**、**PostgreSQL+ES**、**PostgreSQL+pgvector(单库,零 ES)**、
21
+ > **OceanBase(CE v4.4.2.1)+ES** 四套组合均端到端跑通。见 `docs/接入指南.md`。
22
+
23
+ ## 安装与验证(开发)
24
+
25
+ ```bash
26
+ make install # 创建 .venv + 安装开发依赖(可编辑)
27
+ make verify # 全量验证:lint → type → test → build
28
+ make help # 查看全部命令
29
+
30
+ # 解释器需 >= 3.11;如默认 python3 较旧:
31
+ make PYTHON=python3.11 verify
32
+ ```
33
+
34
+ ## 用法
35
+
36
+ ```python
37
+ from zleap_sag import DataEngine, EngineConfig
38
+ from zleap_sag.config import MySQLConfig, ESConfig, LLMConfig, EmbeddingConfig
39
+
40
+ config = EngineConfig(
41
+ mysql=MySQLConfig(user="sag2", password="...", database="sag2"),
42
+ es=ESConfig(hosts=["http://localhost:9200"]),
43
+ llm=LLMConfig(api_key="...", model="qwen3.6-flash"),
44
+ embedding=EmbeddingConfig(model="bge-large-en-v1.5", dimensions=1024),
45
+ )
46
+
47
+ async with DataEngine(config) as engine:
48
+ await engine.init_schema() # 全新库首次建表(幂等,不删表)
49
+ ing = await engine.ingest("doc.md") # 加载单个文档
50
+ await engine.extract() # 抽取事件/实体
51
+ res = await engine.search("Who founded X?", strategy="multi", top_k=10)
52
+ print(len(res.sections))
53
+ ```
54
+
55
+ 也可从环境变量加载配置(`SAG_` 前缀,`__` 表示嵌套):
56
+
57
+ ```bash
58
+ export SAG_MYSQL__USER=sag2 SAG_MYSQL__PASSWORD=... SAG_MYSQL__DATABASE=sag2
59
+ export SAG_ES__HOSTS='["http://localhost:9200"]'
60
+ export SAG_LLM__API_KEY=... SAG_LLM__MODEL=qwen3.6-flash
61
+ export SAG_EMBEDDING__MODEL=bge-large-en-v1.5
62
+ ```
63
+
64
+ ```python
65
+ config = EngineConfig.from_env() # 或 EngineConfig.from_env(env_file=".env")
66
+ ```
67
+
68
+ 统一异常捕获:
69
+
70
+ ```python
71
+ from zleap_sag import SagError # 所有引擎异常的基类
72
+
73
+ try:
74
+ res = await engine.search("...")
75
+ except SagError as e:
76
+ ...
77
+ ```
78
+
79
+ ## ⚠️ 已知限制(v0.2)
80
+
81
+ **多配置隔离(演进中)**:已引入 DI 资源容器(`ResourceContainer` + `ContextVar` 单源解析),
82
+ 但底层部分核心仍复用进程级全局连接(DB/ES)。因此**同一进程内并发多套不同配置**尚未完全隔离,
83
+ 建议一个进程一套配置;完整隔离随核心逐点切换到容器继续推进。
84
+
85
+ - 关系库可切换 **MySQL / PostgreSQL / SQLite / OceanBase**;向量库可切换 **Elasticsearch / pgvector**(单库 PG 可跑完整引擎)。均已真机验证。
86
+ - `ES` 目前取 `hosts` 列表的第一个地址。
87
+ - `ingest` 以**文件路径**为输入(单文档);字符串/只切不存见 `DataEngine.chunk()`。
88
+
89
+ ## 常用命令
90
+
91
+ | 命令 | 作用 |
92
+ |------|------|
93
+ | `make install` | 创建 `.venv` + 安装开发依赖 |
94
+ | `make lint` | ruff 检查 + 格式校验 |
95
+ | `make format` | ruff 自动修复 + 格式化 |
96
+ | `make type` | mypy 类型检查 |
97
+ | `make test` | 单元测试(跳过 integration) |
98
+ | `make test-integration` | 集成测试(需 docker mysql/es) |
99
+ | `make build` | 构建 sdist + wheel 并 `twine check` |
100
+ | `make verify` | lint → type → test → build 全量验证 |
101
+ | `make clean` | 清理构建与缓存产物 |
@@ -0,0 +1,149 @@
1
+ # A generic, single database configuration.
2
+
3
+ [alembic]
4
+ # path to migration scripts.
5
+ # this is typically a path given in POSIX (e.g. forward slashes)
6
+ # format, relative to the token %(here)s which refers to the location of this
7
+ # ini file
8
+ script_location = %(here)s/migrations
9
+
10
+ # template used to generate migration file names; The default value is %%(rev)s_%%(slug)s
11
+ # Uncomment the line below if you want the files to be prepended with date and time
12
+ # see https://alembic.sqlalchemy.org/en/latest/tutorial.html#editing-the-ini-file
13
+ # for all available tokens
14
+ # file_template = %%(year)d_%%(month).2d_%%(day).2d_%%(hour).2d%%(minute).2d-%%(rev)s_%%(slug)s
15
+ # Or organize into date-based subdirectories (requires recursive_version_locations = true)
16
+ # file_template = %%(year)d/%%(month).2d/%%(day).2d_%%(hour).2d%%(minute).2d_%%(second).2d_%%(rev)s_%%(slug)s
17
+
18
+ # sys.path path, will be prepended to sys.path if present.
19
+ # defaults to the current working directory. for multiple paths, the path separator
20
+ # is defined by "path_separator" below.
21
+ prepend_sys_path = .
22
+
23
+
24
+ # timezone to use when rendering the date within the migration file
25
+ # as well as the filename.
26
+ # If specified, requires the tzdata library which can be installed by adding
27
+ # `alembic[tz]` to the pip requirements.
28
+ # string value is passed to ZoneInfo()
29
+ # leave blank for localtime
30
+ # timezone =
31
+
32
+ # max length of characters to apply to the "slug" field
33
+ # truncate_slug_length = 40
34
+
35
+ # set to 'true' to run the environment during
36
+ # the 'revision' command, regardless of autogenerate
37
+ # revision_environment = false
38
+
39
+ # set to 'true' to allow .pyc and .pyo files without
40
+ # a source .py file to be detected as revisions in the
41
+ # versions/ directory
42
+ # sourceless = false
43
+
44
+ # version location specification; This defaults
45
+ # to <script_location>/versions. When using multiple version
46
+ # directories, initial revisions must be specified with --version-path.
47
+ # The path separator used here should be the separator specified by "path_separator"
48
+ # below.
49
+ # version_locations = %(here)s/bar:%(here)s/bat:%(here)s/alembic/versions
50
+
51
+ # path_separator; This indicates what character is used to split lists of file
52
+ # paths, including version_locations and prepend_sys_path within configparser
53
+ # files such as alembic.ini.
54
+ # The default rendered in new alembic.ini files is "os", which uses os.pathsep
55
+ # to provide os-dependent path splitting.
56
+ #
57
+ # Note that in order to support legacy alembic.ini files, this default does NOT
58
+ # take place if path_separator is not present in alembic.ini. If this
59
+ # option is omitted entirely, fallback logic is as follows:
60
+ #
61
+ # 1. Parsing of the version_locations option falls back to using the legacy
62
+ # "version_path_separator" key, which if absent then falls back to the legacy
63
+ # behavior of splitting on spaces and/or commas.
64
+ # 2. Parsing of the prepend_sys_path option falls back to the legacy
65
+ # behavior of splitting on spaces, commas, or colons.
66
+ #
67
+ # Valid values for path_separator are:
68
+ #
69
+ # path_separator = :
70
+ # path_separator = ;
71
+ # path_separator = space
72
+ # path_separator = newline
73
+ #
74
+ # Use os.pathsep. Default configuration used for new projects.
75
+ path_separator = os
76
+
77
+ # set to 'true' to search source files recursively
78
+ # in each "version_locations" directory
79
+ # new in Alembic version 1.10
80
+ # recursive_version_locations = false
81
+
82
+ # the output encoding used when revision files
83
+ # are written from script.py.mako
84
+ # output_encoding = utf-8
85
+
86
+ # database URL. This is consumed by the user-maintained env.py script only.
87
+ # other means of configuring database URLs may be customized within the env.py
88
+ # file.
89
+ sqlalchemy.url = driver://user:pass@localhost/dbname
90
+
91
+
92
+ [post_write_hooks]
93
+ # post_write_hooks defines scripts or Python functions that are run
94
+ # on newly generated revision scripts. See the documentation for further
95
+ # detail and examples
96
+
97
+ # format using "black" - use the console_scripts runner, against the "black" entrypoint
98
+ # hooks = black
99
+ # black.type = console_scripts
100
+ # black.entrypoint = black
101
+ # black.options = -l 79 REVISION_SCRIPT_FILENAME
102
+
103
+ # lint with attempts to fix using "ruff" - use the module runner, against the "ruff" module
104
+ # hooks = ruff
105
+ # ruff.type = module
106
+ # ruff.module = ruff
107
+ # ruff.options = check --fix REVISION_SCRIPT_FILENAME
108
+
109
+ # Alternatively, use the exec runner to execute a binary found on your PATH
110
+ # hooks = ruff
111
+ # ruff.type = exec
112
+ # ruff.executable = ruff
113
+ # ruff.options = check --fix REVISION_SCRIPT_FILENAME
114
+
115
+ # Logging configuration. This is also consumed by the user-maintained
116
+ # env.py script only.
117
+ [loggers]
118
+ keys = root,sqlalchemy,alembic
119
+
120
+ [handlers]
121
+ keys = console
122
+
123
+ [formatters]
124
+ keys = generic
125
+
126
+ [logger_root]
127
+ level = WARNING
128
+ handlers = console
129
+ qualname =
130
+
131
+ [logger_sqlalchemy]
132
+ level = WARNING
133
+ handlers =
134
+ qualname = sqlalchemy.engine
135
+
136
+ [logger_alembic]
137
+ level = INFO
138
+ handlers =
139
+ qualname = alembic
140
+
141
+ [handler_console]
142
+ class = StreamHandler
143
+ args = (sys.stderr,)
144
+ level = NOTSET
145
+ formatter = generic
146
+
147
+ [formatter_generic]
148
+ format = %(levelname)-5.5s [%(name)s] %(message)s
149
+ datefmt = %H:%M:%S
@@ -0,0 +1 @@
1
+ Generic single-database configuration.
@@ -0,0 +1,67 @@
1
+ """Alembic 迁移环境。
2
+
3
+ - target_metadata = 全部 ORM 模型(zleap.sag.db.models 注册到 Base.metadata)。
4
+ - 数据库 URL 来自环境变量 ``SAG_DB_URL``(同步驱动,如 mysql+pymysql / postgresql+psycopg / sqlite),
5
+ 未设置则回退到 alembic.ini 的 sqlalchemy.url。
6
+ - 迁移用同步引擎(运行期用异步;迁移一次性、用同步更简单稳妥)。
7
+ """
8
+
9
+ from __future__ import annotations
10
+
11
+ import os
12
+ from logging.config import fileConfig
13
+
14
+ from sqlalchemy import engine_from_config, pool
15
+
16
+ from alembic import context
17
+
18
+ # 注册全部表到 Base.metadata
19
+ import zleap.sag.db.models # noqa: F401
20
+ from zleap.sag.db.base import Base
21
+
22
+ config = context.config
23
+
24
+ # 允许用 SAG_DB_URL 覆盖 ini 里的连接串
25
+ _env_url = os.getenv("SAG_DB_URL")
26
+ if _env_url:
27
+ config.set_main_option("sqlalchemy.url", _env_url)
28
+
29
+ if config.config_file_name is not None:
30
+ fileConfig(config.config_file_name)
31
+
32
+ target_metadata = Base.metadata
33
+
34
+
35
+ def run_migrations_offline() -> None:
36
+ url = config.get_main_option("sqlalchemy.url")
37
+ context.configure(
38
+ url=url,
39
+ target_metadata=target_metadata,
40
+ literal_binds=True,
41
+ dialect_opts={"paramstyle": "named"},
42
+ compare_type=True,
43
+ )
44
+ with context.begin_transaction():
45
+ context.run_migrations()
46
+
47
+
48
+ def run_migrations_online() -> None:
49
+ connectable = engine_from_config(
50
+ config.get_section(config.config_ini_section, {}),
51
+ prefix="sqlalchemy.",
52
+ poolclass=pool.NullPool,
53
+ )
54
+ with connectable.connect() as connection:
55
+ context.configure(
56
+ connection=connection,
57
+ target_metadata=target_metadata,
58
+ compare_type=True,
59
+ )
60
+ with context.begin_transaction():
61
+ context.run_migrations()
62
+
63
+
64
+ if context.is_offline_mode():
65
+ run_migrations_offline()
66
+ else:
67
+ run_migrations_online()
@@ -0,0 +1,28 @@
1
+ """${message}
2
+
3
+ Revision ID: ${up_revision}
4
+ Revises: ${down_revision | comma,n}
5
+ Create Date: ${create_date}
6
+
7
+ """
8
+ from typing import Sequence, Union
9
+
10
+ from alembic import op
11
+ import sqlalchemy as sa
12
+ ${imports if imports else ""}
13
+
14
+ # revision identifiers, used by Alembic.
15
+ revision: str = ${repr(up_revision)}
16
+ down_revision: Union[str, Sequence[str], None] = ${repr(down_revision)}
17
+ branch_labels: Union[str, Sequence[str], None] = ${repr(branch_labels)}
18
+ depends_on: Union[str, Sequence[str], None] = ${repr(depends_on)}
19
+
20
+
21
+ def upgrade() -> None:
22
+ """Upgrade schema."""
23
+ ${upgrades if upgrades else "pass"}
24
+
25
+
26
+ def downgrade() -> None:
27
+ """Downgrade schema."""
28
+ ${downgrades if downgrades else "pass"}