databao-context-engine 0.1.3__tar.gz → 0.1.5__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- databao_context_engine-0.1.5/PKG-INFO +228 -0
- databao_context_engine-0.1.5/README.md +202 -0
- {databao_context_engine-0.1.3 → databao_context_engine-0.1.5}/pyproject.toml +15 -3
- {databao_context_engine-0.1.3 → databao_context_engine-0.1.5}/src/databao_context_engine/__init__.py +14 -1
- {databao_context_engine-0.1.3 → databao_context_engine-0.1.5}/src/databao_context_engine/build_sources/build_runner.py +7 -7
- {databao_context_engine-0.1.3 → databao_context_engine-0.1.5}/src/databao_context_engine/build_sources/build_wiring.py +8 -10
- {databao_context_engine-0.1.3 → databao_context_engine-0.1.5}/src/databao_context_engine/build_sources/plugin_execution.py +9 -12
- {databao_context_engine-0.1.3 → databao_context_engine-0.1.5}/src/databao_context_engine/cli/add_datasource_config.py +10 -31
- {databao_context_engine-0.1.3 → databao_context_engine-0.1.5}/src/databao_context_engine/cli/commands.py +29 -13
- {databao_context_engine-0.1.3 → databao_context_engine-0.1.5}/src/databao_context_engine/cli/info.py +1 -0
- {databao_context_engine-0.1.3 → databao_context_engine-0.1.5}/src/databao_context_engine/databao_context_engine.py +3 -13
- {databao_context_engine-0.1.3 → databao_context_engine-0.1.5}/src/databao_context_engine/databao_context_project_manager.py +56 -29
- {databao_context_engine-0.1.3 → databao_context_engine-0.1.5}/src/databao_context_engine/datasources/check_config.py +13 -16
- {databao_context_engine-0.1.3 → databao_context_engine-0.1.5}/src/databao_context_engine/datasources/datasource_context.py +21 -24
- databao_context_engine-0.1.5/src/databao_context_engine/datasources/datasource_discovery.py +143 -0
- {databao_context_engine-0.1.3 → databao_context_engine-0.1.5}/src/databao_context_engine/datasources/types.py +53 -42
- {databao_context_engine-0.1.3 → databao_context_engine-0.1.5}/src/databao_context_engine/generate_configs_schemas.py +4 -5
- {databao_context_engine-0.1.3 → databao_context_engine-0.1.5}/src/databao_context_engine/introspection/property_extract.py +61 -71
- databao_context_engine-0.1.5/src/databao_context_engine/llm/__init__.py +10 -0
- databao_context_engine-0.1.5/src/databao_context_engine/llm/api.py +57 -0
- {databao_context_engine-0.1.3 → databao_context_engine-0.1.5}/src/databao_context_engine/llm/descriptions/ollama.py +1 -3
- {databao_context_engine-0.1.3 → databao_context_engine-0.1.5}/src/databao_context_engine/llm/factory.py +5 -2
- {databao_context_engine-0.1.3 → databao_context_engine-0.1.5}/src/databao_context_engine/llm/install.py +13 -10
- {databao_context_engine-0.1.3 → databao_context_engine-0.1.5}/src/databao_context_engine/llm/runtime.py +3 -5
- {databao_context_engine-0.1.3 → databao_context_engine-0.1.5}/src/databao_context_engine/mcp/mcp_server.py +9 -8
- {databao_context_engine-0.1.3 → databao_context_engine-0.1.5}/src/databao_context_engine/plugin_loader.py +6 -7
- {databao_context_engine-0.1.3 → databao_context_engine-0.1.5}/src/databao_context_engine/pluginlib/build_plugin.py +0 -33
- {databao_context_engine-0.1.3/src/databao_context_engine/plugins → databao_context_engine-0.1.5/src/databao_context_engine/plugins/databases/athena}/athena_db_plugin.py +3 -3
- {databao_context_engine-0.1.3/src/databao_context_engine/plugins/databases → databao_context_engine-0.1.5/src/databao_context_engine/plugins/databases/athena}/athena_introspector.py +3 -6
- {databao_context_engine-0.1.3/src/databao_context_engine/plugins → databao_context_engine-0.1.5/src/databao_context_engine/plugins/databases}/base_db_plugin.py +6 -5
- {databao_context_engine-0.1.3 → databao_context_engine-0.1.5}/src/databao_context_engine/plugins/databases/base_introspector.py +11 -14
- {databao_context_engine-0.1.3/src/databao_context_engine/plugins → databao_context_engine-0.1.5/src/databao_context_engine/plugins/databases/clickhouse}/clickhouse_db_plugin.py +3 -3
- {databao_context_engine-0.1.3/src/databao_context_engine/plugins/databases → databao_context_engine-0.1.5/src/databao_context_engine/plugins/databases/clickhouse}/clickhouse_introspector.py +3 -6
- databao_context_engine-0.1.5/src/databao_context_engine/plugins/databases/duckdb/duckdb_db_plugin.py +12 -0
- {databao_context_engine-0.1.3/src/databao_context_engine/plugins/databases → databao_context_engine-0.1.5/src/databao_context_engine/plugins/databases/duckdb}/duckdb_introspector.py +6 -11
- {databao_context_engine-0.1.3/src/databao_context_engine/plugins → databao_context_engine-0.1.5/src/databao_context_engine/plugins/databases/mssql}/mssql_db_plugin.py +3 -3
- {databao_context_engine-0.1.3/src/databao_context_engine/plugins/databases → databao_context_engine-0.1.5/src/databao_context_engine/plugins/databases/mssql}/mssql_introspector.py +11 -12
- {databao_context_engine-0.1.3/src/databao_context_engine/plugins → databao_context_engine-0.1.5/src/databao_context_engine/plugins/databases/mysql}/mysql_db_plugin.py +3 -3
- {databao_context_engine-0.1.3/src/databao_context_engine/plugins/databases → databao_context_engine-0.1.5/src/databao_context_engine/plugins/databases/mysql}/mysql_introspector.py +9 -9
- databao_context_engine-0.1.5/src/databao_context_engine/plugins/databases/postgresql/postgresql_db_plugin.py +15 -0
- {databao_context_engine-0.1.3/src/databao_context_engine/plugins/databases → databao_context_engine-0.1.5/src/databao_context_engine/plugins/databases/postgresql}/postgresql_introspector.py +9 -16
- databao_context_engine-0.1.5/src/databao_context_engine/plugins/databases/snowflake/snowflake_db_plugin.py +15 -0
- {databao_context_engine-0.1.3/src/databao_context_engine/plugins/databases → databao_context_engine-0.1.5/src/databao_context_engine/plugins/databases/snowflake}/snowflake_introspector.py +9 -10
- databao_context_engine-0.1.5/src/databao_context_engine/plugins/databases/sqlite/sqlite_db_plugin.py +12 -0
- databao_context_engine-0.1.5/src/databao_context_engine/plugins/databases/sqlite/sqlite_introspector.py +241 -0
- databao_context_engine-0.1.5/src/databao_context_engine/plugins/duckdb_tools.py +18 -0
- {databao_context_engine-0.1.3/src/databao_context_engine/plugins → databao_context_engine-0.1.5/src/databao_context_engine/plugins/files}/unstructured_files_plugin.py +1 -1
- {databao_context_engine-0.1.3 → databao_context_engine-0.1.5}/src/databao_context_engine/plugins/plugin_loader.py +12 -15
- {databao_context_engine-0.1.3 → databao_context_engine-0.1.5}/src/databao_context_engine/plugins/resources/parquet_introspector.py +6 -17
- {databao_context_engine-0.1.3/src/databao_context_engine/plugins → databao_context_engine-0.1.5/src/databao_context_engine/plugins/resources}/parquet_plugin.py +1 -3
- databao_context_engine-0.1.5/src/databao_context_engine/project/__init__.py +0 -0
- {databao_context_engine-0.1.3 → databao_context_engine-0.1.5}/src/databao_context_engine/project/info.py +3 -0
- {databao_context_engine-0.1.3 → databao_context_engine-0.1.5}/src/databao_context_engine/project/layout.py +12 -13
- databao_context_engine-0.1.5/src/databao_context_engine/retrieve_embeddings/retrieve_runner.py +17 -0
- {databao_context_engine-0.1.3 → databao_context_engine-0.1.5}/src/databao_context_engine/retrieve_embeddings/retrieve_service.py +13 -6
- {databao_context_engine-0.1.3 → databao_context_engine-0.1.5}/src/databao_context_engine/retrieve_embeddings/retrieve_wiring.py +4 -7
- databao_context_engine-0.1.5/src/databao_context_engine/serialization/__init__.py +0 -0
- {databao_context_engine-0.1.3 → databao_context_engine-0.1.5}/src/databao_context_engine/serialization/yaml.py +5 -5
- databao_context_engine-0.1.5/src/databao_context_engine/services/__init__.py +0 -0
- databao_context_engine-0.1.5/src/databao_context_engine/storage/__init__.py +0 -0
- databao_context_engine-0.1.5/src/databao_context_engine/storage/exceptions/__init__.py +0 -0
- {databao_context_engine-0.1.3 → databao_context_engine-0.1.5}/src/databao_context_engine/storage/migrate.py +1 -1
- databao_context_engine-0.1.5/src/databao_context_engine/storage/repositories/__init__.py +0 -0
- {databao_context_engine-0.1.3 → databao_context_engine-0.1.5}/src/databao_context_engine/storage/repositories/vector_search_repository.py +18 -6
- databao_context_engine-0.1.5/src/databao_context_engine/system/__init__.py +0 -0
- databao_context_engine-0.1.5/src/databao_context_engine/templating/__init__.py +0 -0
- databao_context_engine-0.1.3/PKG-INFO +0 -75
- databao_context_engine-0.1.3/README.md +0 -49
- databao_context_engine-0.1.3/src/databao_context_engine/datasources/add_config.py +0 -34
- databao_context_engine-0.1.3/src/databao_context_engine/datasources/datasource_discovery.py +0 -142
- databao_context_engine-0.1.3/src/databao_context_engine/mcp/all_results_tool.py +0 -5
- databao_context_engine-0.1.3/src/databao_context_engine/mcp/retrieve_tool.py +0 -14
- databao_context_engine-0.1.3/src/databao_context_engine/plugins/duckdb_db_plugin.py +0 -12
- databao_context_engine-0.1.3/src/databao_context_engine/plugins/postgresql_db_plugin.py +0 -12
- databao_context_engine-0.1.3/src/databao_context_engine/plugins/snowflake_db_plugin.py +0 -12
- databao_context_engine-0.1.3/src/databao_context_engine/retrieve_embeddings/export_results.py +0 -12
- databao_context_engine-0.1.3/src/databao_context_engine/retrieve_embeddings/retrieve_runner.py +0 -30
- {databao_context_engine-0.1.3 → databao_context_engine-0.1.5}/src/databao_context_engine/build_sources/__init__.py +0 -0
- {databao_context_engine-0.1.3 → databao_context_engine-0.1.5}/src/databao_context_engine/build_sources/build_service.py +0 -0
- {databao_context_engine-0.1.3 → databao_context_engine-0.1.5}/src/databao_context_engine/build_sources/export_results.py +0 -0
- {databao_context_engine-0.1.3 → databao_context_engine-0.1.5}/src/databao_context_engine/cli/__init__.py +0 -0
- {databao_context_engine-0.1.3 → databao_context_engine-0.1.5}/src/databao_context_engine/cli/datasources.py +0 -0
- {databao_context_engine-0.1.3 → databao_context_engine-0.1.5}/src/databao_context_engine/config/__init__.py +0 -0
- {databao_context_engine-0.1.3 → databao_context_engine-0.1.5}/src/databao_context_engine/config/log_config.yaml +0 -0
- {databao_context_engine-0.1.3 → databao_context_engine-0.1.5}/src/databao_context_engine/config/logging.py +0 -0
- {databao_context_engine-0.1.3 → databao_context_engine-0.1.5}/src/databao_context_engine/datasources/__init__.py +0 -0
- {databao_context_engine-0.1.3 → databao_context_engine-0.1.5}/src/databao_context_engine/event_journal/__init__.py +0 -0
- {databao_context_engine-0.1.3 → databao_context_engine-0.1.5}/src/databao_context_engine/event_journal/writer.py +0 -0
- {databao_context_engine-0.1.3 → databao_context_engine-0.1.5}/src/databao_context_engine/init_project.py +0 -0
- {databao_context_engine-0.1.3 → databao_context_engine-0.1.5}/src/databao_context_engine/introspection/__init__.py +0 -0
- {databao_context_engine-0.1.3 → databao_context_engine-0.1.5}/src/databao_context_engine/llm/config.py +0 -0
- {databao_context_engine-0.1.3/src/databao_context_engine/llm → databao_context_engine-0.1.5/src/databao_context_engine/llm/descriptions}/__init__.py +0 -0
- {databao_context_engine-0.1.3 → databao_context_engine-0.1.5}/src/databao_context_engine/llm/descriptions/provider.py +0 -0
- {databao_context_engine-0.1.3/src/databao_context_engine/llm/descriptions → databao_context_engine-0.1.5/src/databao_context_engine/llm/embeddings}/__init__.py +0 -0
- {databao_context_engine-0.1.3 → databao_context_engine-0.1.5}/src/databao_context_engine/llm/embeddings/ollama.py +0 -0
- {databao_context_engine-0.1.3 → databao_context_engine-0.1.5}/src/databao_context_engine/llm/embeddings/provider.py +0 -0
- {databao_context_engine-0.1.3 → databao_context_engine-0.1.5}/src/databao_context_engine/llm/errors.py +0 -0
- {databao_context_engine-0.1.3 → databao_context_engine-0.1.5}/src/databao_context_engine/llm/service.py +0 -0
- {databao_context_engine-0.1.3 → databao_context_engine-0.1.5}/src/databao_context_engine/main.py +0 -0
- {databao_context_engine-0.1.3/src/databao_context_engine/llm/embeddings → databao_context_engine-0.1.5/src/databao_context_engine/mcp}/__init__.py +0 -0
- {databao_context_engine-0.1.3 → databao_context_engine-0.1.5}/src/databao_context_engine/mcp/mcp_runner.py +0 -0
- {databao_context_engine-0.1.3/src/databao_context_engine/mcp → databao_context_engine-0.1.5/src/databao_context_engine/pluginlib}/__init__.py +0 -0
- {databao_context_engine-0.1.3 → databao_context_engine-0.1.5}/src/databao_context_engine/pluginlib/config.py +0 -0
- {databao_context_engine-0.1.3 → databao_context_engine-0.1.5}/src/databao_context_engine/pluginlib/plugin_utils.py +0 -0
- {databao_context_engine-0.1.3/src/databao_context_engine/pluginlib → databao_context_engine-0.1.5/src/databao_context_engine/plugins}/__init__.py +0 -0
- {databao_context_engine-0.1.3/src/databao_context_engine/plugins → databao_context_engine-0.1.5/src/databao_context_engine/plugins/databases}/__init__.py +0 -0
- {databao_context_engine-0.1.3/src/databao_context_engine/plugins/databases → databao_context_engine-0.1.5/src/databao_context_engine/plugins/databases/athena}/__init__.py +0 -0
- {databao_context_engine-0.1.3/src/databao_context_engine/plugins/resources → databao_context_engine-0.1.5/src/databao_context_engine/plugins/databases/clickhouse}/__init__.py +0 -0
- {databao_context_engine-0.1.3 → databao_context_engine-0.1.5}/src/databao_context_engine/plugins/databases/database_chunker.py +0 -0
- {databao_context_engine-0.1.3 → databao_context_engine-0.1.5}/src/databao_context_engine/plugins/databases/databases_types.py +0 -0
- {databao_context_engine-0.1.3/src/databao_context_engine/project → databao_context_engine-0.1.5/src/databao_context_engine/plugins/databases/duckdb}/__init__.py +0 -0
- {databao_context_engine-0.1.3 → databao_context_engine-0.1.5}/src/databao_context_engine/plugins/databases/introspection_model_builder.py +1 -1
- {databao_context_engine-0.1.3 → databao_context_engine-0.1.5}/src/databao_context_engine/plugins/databases/introspection_scope.py +0 -0
- {databao_context_engine-0.1.3 → databao_context_engine-0.1.5}/src/databao_context_engine/plugins/databases/introspection_scope_matcher.py +0 -0
- {databao_context_engine-0.1.3/src/databao_context_engine/serialization → databao_context_engine-0.1.5/src/databao_context_engine/plugins/databases/mssql}/__init__.py +0 -0
- {databao_context_engine-0.1.3/src/databao_context_engine/services → databao_context_engine-0.1.5/src/databao_context_engine/plugins/databases/mysql}/__init__.py +0 -0
- {databao_context_engine-0.1.3/src/databao_context_engine/storage → databao_context_engine-0.1.5/src/databao_context_engine/plugins/databases/postgresql}/__init__.py +0 -0
- {databao_context_engine-0.1.3/src/databao_context_engine/storage/exceptions → databao_context_engine-0.1.5/src/databao_context_engine/plugins/databases/snowflake}/__init__.py +0 -0
- {databao_context_engine-0.1.3/src/databao_context_engine/storage/repositories → databao_context_engine-0.1.5/src/databao_context_engine/plugins/databases/sqlite}/__init__.py +0 -0
- {databao_context_engine-0.1.3/src/databao_context_engine/system → databao_context_engine-0.1.5/src/databao_context_engine/plugins/files}/__init__.py +0 -0
- {databao_context_engine-0.1.3/src/databao_context_engine/templating → databao_context_engine-0.1.5/src/databao_context_engine/plugins/resources}/__init__.py +0 -0
- {databao_context_engine-0.1.3 → databao_context_engine-0.1.5}/src/databao_context_engine/plugins/resources/parquet_chunker.py +0 -0
- {databao_context_engine-0.1.3 → databao_context_engine-0.1.5}/src/databao_context_engine/project/init_project.py +0 -0
- {databao_context_engine-0.1.3 → databao_context_engine-0.1.5}/src/databao_context_engine/project/project_config.py +0 -0
- {databao_context_engine-0.1.3 → databao_context_engine-0.1.5}/src/databao_context_engine/project/resources/examples/src/databases/example_postgres.yaml +0 -0
- {databao_context_engine-0.1.3 → databao_context_engine-0.1.5}/src/databao_context_engine/project/resources/examples/src/files/documentation.md +0 -0
- {databao_context_engine-0.1.3 → databao_context_engine-0.1.5}/src/databao_context_engine/project/resources/examples/src/files/notes.txt +0 -0
- {databao_context_engine-0.1.3 → databao_context_engine-0.1.5}/src/databao_context_engine/retrieve_embeddings/__init__.py +0 -0
- {databao_context_engine-0.1.3 → databao_context_engine-0.1.5}/src/databao_context_engine/services/chunk_embedding_service.py +0 -0
- {databao_context_engine-0.1.3 → databao_context_engine-0.1.5}/src/databao_context_engine/services/embedding_shard_resolver.py +0 -0
- {databao_context_engine-0.1.3 → databao_context_engine-0.1.5}/src/databao_context_engine/services/factories.py +0 -0
- {databao_context_engine-0.1.3 → databao_context_engine-0.1.5}/src/databao_context_engine/services/models.py +0 -0
- {databao_context_engine-0.1.3 → databao_context_engine-0.1.5}/src/databao_context_engine/services/persistence_service.py +0 -0
- {databao_context_engine-0.1.3 → databao_context_engine-0.1.5}/src/databao_context_engine/services/table_name_policy.py +0 -0
- {databao_context_engine-0.1.3 → databao_context_engine-0.1.5}/src/databao_context_engine/storage/connection.py +0 -0
- {databao_context_engine-0.1.3 → databao_context_engine-0.1.5}/src/databao_context_engine/storage/exceptions/exceptions.py +0 -0
- {databao_context_engine-0.1.3 → databao_context_engine-0.1.5}/src/databao_context_engine/storage/migrations/V01__init.sql +0 -0
- {databao_context_engine-0.1.3 → databao_context_engine-0.1.5}/src/databao_context_engine/storage/models.py +0 -0
- {databao_context_engine-0.1.3 → databao_context_engine-0.1.5}/src/databao_context_engine/storage/repositories/chunk_repository.py +0 -0
- {databao_context_engine-0.1.3 → databao_context_engine-0.1.5}/src/databao_context_engine/storage/repositories/embedding_model_registry_repository.py +0 -0
- {databao_context_engine-0.1.3 → databao_context_engine-0.1.5}/src/databao_context_engine/storage/repositories/embedding_repository.py +0 -0
- {databao_context_engine-0.1.3 → databao_context_engine-0.1.5}/src/databao_context_engine/storage/repositories/factories.py +0 -0
- {databao_context_engine-0.1.3 → databao_context_engine-0.1.5}/src/databao_context_engine/storage/transaction.py +0 -0
- {databao_context_engine-0.1.3 → databao_context_engine-0.1.5}/src/databao_context_engine/system/properties.py +0 -0
- {databao_context_engine-0.1.3 → databao_context_engine-0.1.5}/src/databao_context_engine/templating/renderer.py +0 -0
|
@@ -0,0 +1,228 @@
|
|
|
1
|
+
Metadata-Version: 2.3
|
|
2
|
+
Name: databao-context-engine
|
|
3
|
+
Version: 0.1.5
|
|
4
|
+
Summary: Semantic context for your LLMs — generated automatically
|
|
5
|
+
Requires-Dist: click>=8.3.0
|
|
6
|
+
Requires-Dist: duckdb>=1.4.3
|
|
7
|
+
Requires-Dist: pyyaml>=6.0.3
|
|
8
|
+
Requires-Dist: requests>=2.32.5
|
|
9
|
+
Requires-Dist: mcp>=1.23.3
|
|
10
|
+
Requires-Dist: pydantic>=2.12.4
|
|
11
|
+
Requires-Dist: jinja2>=3.1.6
|
|
12
|
+
Requires-Dist: pyathena>=3.25.0 ; extra == 'athena'
|
|
13
|
+
Requires-Dist: clickhouse-connect>=0.10.0 ; extra == 'clickhouse'
|
|
14
|
+
Requires-Dist: mssql-python>=1.0.0 ; extra == 'mssql'
|
|
15
|
+
Requires-Dist: pymysql>=1.1.2 ; extra == 'mysql'
|
|
16
|
+
Requires-Dist: asyncpg>=0.31.0 ; extra == 'postgresql'
|
|
17
|
+
Requires-Dist: snowflake-connector-python>=4.2.0 ; extra == 'snowflake'
|
|
18
|
+
Requires-Python: >=3.12
|
|
19
|
+
Provides-Extra: athena
|
|
20
|
+
Provides-Extra: clickhouse
|
|
21
|
+
Provides-Extra: mssql
|
|
22
|
+
Provides-Extra: mysql
|
|
23
|
+
Provides-Extra: postgresql
|
|
24
|
+
Provides-Extra: snowflake
|
|
25
|
+
Description-Content-Type: text/markdown
|
|
26
|
+
|
|
27
|
+
[](https://github.com/JetBrains#jetbrains-on-github)
|
|
28
|
+
[](https://pypi.org/project/databao-context-engine)
|
|
29
|
+
[](https://github.com/JetBrains/databao-context-engine?tab=License-1-ov-file)
|
|
30
|
+
|
|
31
|
+
[//]: # ([](https://pypi.org/project/databao-context-engine/))
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
<h1 align="center">Databao Context Engine</h1>
|
|
35
|
+
<p align="center">
|
|
36
|
+
<b>Semantic context for your LLMs — generated automatically.</b><br/>
|
|
37
|
+
No more copying schemas. No manual documentation. Just accurate answers.
|
|
38
|
+
</p>
|
|
39
|
+
<p align="center">
|
|
40
|
+
<a href="https://databao.app">Website</a> •
|
|
41
|
+
<a href="#quickstart">Quickstart</a> •
|
|
42
|
+
<a href="#supported-data-sources">Data Sources</a> •
|
|
43
|
+
<a href="#contributing">Contributing</a>
|
|
44
|
+
</p>
|
|
45
|
+
|
|
46
|
+
---
|
|
47
|
+
|
|
48
|
+
## What is Databao Context Engine?
|
|
49
|
+
|
|
50
|
+
Databao Context Engine is a CLI tool that **automatically generates governed semantic context** from your databases, BI tools, documents, and spreadsheets.
|
|
51
|
+
|
|
52
|
+
Integrate it with any LLM to deliver **accurate, context-aware answers** — without copying schemas or writing documentation by hand.
|
|
53
|
+
|
|
54
|
+
```
|
|
55
|
+
Your data sources → Context Engine → Unified semantic graph → Any LLM
|
|
56
|
+
```
|
|
57
|
+
|
|
58
|
+
## Why choose Databao Context Engine?
|
|
59
|
+
|
|
60
|
+
| Feature | What it means for you |
|
|
61
|
+
|----------------------------|----------------------------------------------------------------|
|
|
62
|
+
| **Auto-generated context** | Extracts schemas, relationships, and semantics automatically |
|
|
63
|
+
| **Runs locally** | Your data never leaves your environment |
|
|
64
|
+
| **MCP integration** | Works with Claude Desktop, Cursor, and any MCP-compatible tool |
|
|
65
|
+
| **Multiple sources** | Databases, dbt projects, spreadsheets, documents |
|
|
66
|
+
| **Built-in benchmarks** | Measure and improve context quality over time |
|
|
67
|
+
| **LLM agnostic** | OpenAI, Anthropic, Ollama, Gemini — use any model |
|
|
68
|
+
| **Governed & versioned** | Track, version, and share context across your team |
|
|
69
|
+
| **Dynamic or static** | Serve context via MCP server or export as artifact |
|
|
70
|
+
|
|
71
|
+
## Installation
|
|
72
|
+
|
|
73
|
+
Databao Context Engine is [available on PyPI](https://pypi.org/project/databao-context-engine/) and can be installed with uv, pip, or another package manage.
|
|
74
|
+
|
|
75
|
+
### Using uv
|
|
76
|
+
|
|
77
|
+
1. Install Databao Context Engine:
|
|
78
|
+
|
|
79
|
+
```bash
|
|
80
|
+
uv tool install databao-context-engine
|
|
81
|
+
```
|
|
82
|
+
|
|
83
|
+
1. Add it to your PATH:
|
|
84
|
+
|
|
85
|
+
```bash
|
|
86
|
+
uv tool update-shell
|
|
87
|
+
```
|
|
88
|
+
|
|
89
|
+
1. Verify the installation:
|
|
90
|
+
|
|
91
|
+
```bash
|
|
92
|
+
dce --help
|
|
93
|
+
```
|
|
94
|
+
|
|
95
|
+
### Using pip
|
|
96
|
+
|
|
97
|
+
1. Install Databao Context Engine:
|
|
98
|
+
|
|
99
|
+
```bash
|
|
100
|
+
pip install databao-context-engine
|
|
101
|
+
```
|
|
102
|
+
|
|
103
|
+
1. Verify the installation:
|
|
104
|
+
|
|
105
|
+
```bash
|
|
106
|
+
dce --help
|
|
107
|
+
```
|
|
108
|
+
|
|
109
|
+
## Supported data sources
|
|
110
|
+
|
|
111
|
+
* <img src="https://cdn.simpleicons.org/postgresql/316192" width="16" height="16" alt=""> PostgreSQL
|
|
112
|
+
* <img src="https://cdn.simpleicons.org/mysql/4479A1" width="16" height="16" alt=""> MySQL
|
|
113
|
+
* <img src="https://cdn.simpleicons.org/sqlite/003B57" width="16" height="16" alt=""> SQLite
|
|
114
|
+
* <img src="https://cdn.simpleicons.org/duckdb/FFF000" width="16" height="16" alt=""> DuckDB
|
|
115
|
+
* <img src="https://cdn.simpleicons.org/dbt/FF694B" width="16" height="16" alt=""> dbt projects
|
|
116
|
+
* 📄 Documents & spreadsheets *(coming soon)*
|
|
117
|
+
|
|
118
|
+
## Supported LLMs
|
|
119
|
+
|
|
120
|
+
| Provider | Configuration |
|
|
121
|
+
|---------------|----------------------------------------------|
|
|
122
|
+
| **Ollama** | `languageModel: OLLAMA`: runs locally, free |
|
|
123
|
+
| **OpenAI** | `languageModel: OPENAI`: requires an API key |
|
|
124
|
+
| **Anthropic** | `languageModel: CLAUDE`: requires an API key |
|
|
125
|
+
| **Google** | `languageModel: GEMINI`: requires an API key |
|
|
126
|
+
|
|
127
|
+
## Quickstart
|
|
128
|
+
|
|
129
|
+
### 1. Create a project
|
|
130
|
+
|
|
131
|
+
1. Create a new directory for your project and navigate to it:
|
|
132
|
+
|
|
133
|
+
```bash
|
|
134
|
+
mkdir dce-project && cd dce-project
|
|
135
|
+
```
|
|
136
|
+
|
|
137
|
+
1. Initialize a new project:
|
|
138
|
+
|
|
139
|
+
```bash
|
|
140
|
+
dce init
|
|
141
|
+
```
|
|
142
|
+
|
|
143
|
+
### 2. Configure data sources
|
|
144
|
+
|
|
145
|
+
1. When prompted, agree to create a new datasource.
|
|
146
|
+
You can also use the `dce datasource add` command.
|
|
147
|
+
|
|
148
|
+
1. Provide the data source type and its name.
|
|
149
|
+
|
|
150
|
+
1. Open the config file that was created for you in your editor and fill in the connection details.
|
|
151
|
+
|
|
152
|
+
1. Repeat these steps for all data sources you want to include in your project.
|
|
153
|
+
|
|
154
|
+
1. If you have data in Markdown or text files,
|
|
155
|
+
you can add them to the `dce/src/files` directory.
|
|
156
|
+
|
|
157
|
+
### 3. Build context
|
|
158
|
+
|
|
159
|
+
1. To build the context, run the following command:
|
|
160
|
+
|
|
161
|
+
```bash
|
|
162
|
+
dce build
|
|
163
|
+
```
|
|
164
|
+
|
|
165
|
+
### 4. Use Context with Your LLM
|
|
166
|
+
|
|
167
|
+
**Option A: Dynamic via MCP Server**
|
|
168
|
+
|
|
169
|
+
Databao Context Engine exposes the context through a local MCP Server, so your agent can access the latest context at runtime.
|
|
170
|
+
|
|
171
|
+
1. In **Claude Desktop**, **Cursor**, or another MCP-compatible agent, add the following configuration.
|
|
172
|
+
Replace `dce-project/` with the path to your project directory:
|
|
173
|
+
|
|
174
|
+
```json
|
|
175
|
+
# claude_desktop_config.json, mcp.json, or similar
|
|
176
|
+
|
|
177
|
+
{
|
|
178
|
+
"mcpServers": {
|
|
179
|
+
"dce": {
|
|
180
|
+
"command": "dce mcp",
|
|
181
|
+
"args": ["--project-dir", "dce-project/"]
|
|
182
|
+
}
|
|
183
|
+
}
|
|
184
|
+
}
|
|
185
|
+
```
|
|
186
|
+
|
|
187
|
+
1. Save the file and restart your agent.
|
|
188
|
+
|
|
189
|
+
1. Open a new chat, in the chat window, select the `dce` server, and ask questions related to your project context.
|
|
190
|
+
|
|
191
|
+
**Option B: Static artifact**
|
|
192
|
+
|
|
193
|
+
Even if you don’t have Claude or Cursor installed on your local machine,
|
|
194
|
+
you can still use the context built by Databao Context Engine by pasting it directly into your chat with an AI assistant.
|
|
195
|
+
|
|
196
|
+
1. Navigate to `dce-project/output/` and open the directory with the latest run.
|
|
197
|
+
|
|
198
|
+
1. Attach the `all_results.yaml` file to your chat with the AI assistant or copy and paste its contents into your chat.
|
|
199
|
+
|
|
200
|
+
## Contributing
|
|
201
|
+
|
|
202
|
+
We’d love your help! Here’s how to get involved:
|
|
203
|
+
|
|
204
|
+
- ⭐ **Star this repo** — it helps others find us!
|
|
205
|
+
- 🐛 **Found a bug?** [Open an issue](https://github.com/JetBrains/databao-context-engine/issues)
|
|
206
|
+
- 💡 **Have an idea?** We’re all ears — create a feature request
|
|
207
|
+
- 👍 **Upvote issues** you care about — helps us prioritize
|
|
208
|
+
- 🔧 **Submit a PR**
|
|
209
|
+
- 📝 **Improve docs** — typos, examples, tutorials — everything helps!
|
|
210
|
+
|
|
211
|
+
New to open source? No worries! We're friendly and happy to help you get started. 🌱
|
|
212
|
+
|
|
213
|
+
For more details, see [CONTRIBUTING](CONTRIBUTING.md).
|
|
214
|
+
|
|
215
|
+
## 📄 License
|
|
216
|
+
|
|
217
|
+
Apache 2.0 — use it however you want. See the [LICENSE](LICENSE.md) file for details.
|
|
218
|
+
|
|
219
|
+
---
|
|
220
|
+
|
|
221
|
+
<p align="center">
|
|
222
|
+
<b>Like Databao Context Engine?</b> Give us a ⭐ — it means a lot!
|
|
223
|
+
</p>
|
|
224
|
+
|
|
225
|
+
<p align="center">
|
|
226
|
+
<a href="https://databao.app">Website</a> •
|
|
227
|
+
<a href="https://discord.gg/databao">Discord</a>
|
|
228
|
+
</p>
|
|
@@ -0,0 +1,202 @@
|
|
|
1
|
+
[](https://github.com/JetBrains#jetbrains-on-github)
|
|
2
|
+
[](https://pypi.org/project/databao-context-engine)
|
|
3
|
+
[](https://github.com/JetBrains/databao-context-engine?tab=License-1-ov-file)
|
|
4
|
+
|
|
5
|
+
[//]: # ([](https://pypi.org/project/databao-context-engine/))
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
<h1 align="center">Databao Context Engine</h1>
|
|
9
|
+
<p align="center">
|
|
10
|
+
<b>Semantic context for your LLMs — generated automatically.</b><br/>
|
|
11
|
+
No more copying schemas. No manual documentation. Just accurate answers.
|
|
12
|
+
</p>
|
|
13
|
+
<p align="center">
|
|
14
|
+
<a href="https://databao.app">Website</a> •
|
|
15
|
+
<a href="#quickstart">Quickstart</a> •
|
|
16
|
+
<a href="#supported-data-sources">Data Sources</a> •
|
|
17
|
+
<a href="#contributing">Contributing</a>
|
|
18
|
+
</p>
|
|
19
|
+
|
|
20
|
+
---
|
|
21
|
+
|
|
22
|
+
## What is Databao Context Engine?
|
|
23
|
+
|
|
24
|
+
Databao Context Engine is a CLI tool that **automatically generates governed semantic context** from your databases, BI tools, documents, and spreadsheets.
|
|
25
|
+
|
|
26
|
+
Integrate it with any LLM to deliver **accurate, context-aware answers** — without copying schemas or writing documentation by hand.
|
|
27
|
+
|
|
28
|
+
```
|
|
29
|
+
Your data sources → Context Engine → Unified semantic graph → Any LLM
|
|
30
|
+
```
|
|
31
|
+
|
|
32
|
+
## Why choose Databao Context Engine?
|
|
33
|
+
|
|
34
|
+
| Feature | What it means for you |
|
|
35
|
+
|----------------------------|----------------------------------------------------------------|
|
|
36
|
+
| **Auto-generated context** | Extracts schemas, relationships, and semantics automatically |
|
|
37
|
+
| **Runs locally** | Your data never leaves your environment |
|
|
38
|
+
| **MCP integration** | Works with Claude Desktop, Cursor, and any MCP-compatible tool |
|
|
39
|
+
| **Multiple sources** | Databases, dbt projects, spreadsheets, documents |
|
|
40
|
+
| **Built-in benchmarks** | Measure and improve context quality over time |
|
|
41
|
+
| **LLM agnostic** | OpenAI, Anthropic, Ollama, Gemini — use any model |
|
|
42
|
+
| **Governed & versioned** | Track, version, and share context across your team |
|
|
43
|
+
| **Dynamic or static** | Serve context via MCP server or export as artifact |
|
|
44
|
+
|
|
45
|
+
## Installation
|
|
46
|
+
|
|
47
|
+
Databao Context Engine is [available on PyPI](https://pypi.org/project/databao-context-engine/) and can be installed with uv, pip, or another package manage.
|
|
48
|
+
|
|
49
|
+
### Using uv
|
|
50
|
+
|
|
51
|
+
1. Install Databao Context Engine:
|
|
52
|
+
|
|
53
|
+
```bash
|
|
54
|
+
uv tool install databao-context-engine
|
|
55
|
+
```
|
|
56
|
+
|
|
57
|
+
1. Add it to your PATH:
|
|
58
|
+
|
|
59
|
+
```bash
|
|
60
|
+
uv tool update-shell
|
|
61
|
+
```
|
|
62
|
+
|
|
63
|
+
1. Verify the installation:
|
|
64
|
+
|
|
65
|
+
```bash
|
|
66
|
+
dce --help
|
|
67
|
+
```
|
|
68
|
+
|
|
69
|
+
### Using pip
|
|
70
|
+
|
|
71
|
+
1. Install Databao Context Engine:
|
|
72
|
+
|
|
73
|
+
```bash
|
|
74
|
+
pip install databao-context-engine
|
|
75
|
+
```
|
|
76
|
+
|
|
77
|
+
1. Verify the installation:
|
|
78
|
+
|
|
79
|
+
```bash
|
|
80
|
+
dce --help
|
|
81
|
+
```
|
|
82
|
+
|
|
83
|
+
## Supported data sources
|
|
84
|
+
|
|
85
|
+
* <img src="https://cdn.simpleicons.org/postgresql/316192" width="16" height="16" alt=""> PostgreSQL
|
|
86
|
+
* <img src="https://cdn.simpleicons.org/mysql/4479A1" width="16" height="16" alt=""> MySQL
|
|
87
|
+
* <img src="https://cdn.simpleicons.org/sqlite/003B57" width="16" height="16" alt=""> SQLite
|
|
88
|
+
* <img src="https://cdn.simpleicons.org/duckdb/FFF000" width="16" height="16" alt=""> DuckDB
|
|
89
|
+
* <img src="https://cdn.simpleicons.org/dbt/FF694B" width="16" height="16" alt=""> dbt projects
|
|
90
|
+
* 📄 Documents & spreadsheets *(coming soon)*
|
|
91
|
+
|
|
92
|
+
## Supported LLMs
|
|
93
|
+
|
|
94
|
+
| Provider | Configuration |
|
|
95
|
+
|---------------|----------------------------------------------|
|
|
96
|
+
| **Ollama** | `languageModel: OLLAMA`: runs locally, free |
|
|
97
|
+
| **OpenAI** | `languageModel: OPENAI`: requires an API key |
|
|
98
|
+
| **Anthropic** | `languageModel: CLAUDE`: requires an API key |
|
|
99
|
+
| **Google** | `languageModel: GEMINI`: requires an API key |
|
|
100
|
+
|
|
101
|
+
## Quickstart
|
|
102
|
+
|
|
103
|
+
### 1. Create a project
|
|
104
|
+
|
|
105
|
+
1. Create a new directory for your project and navigate to it:
|
|
106
|
+
|
|
107
|
+
```bash
|
|
108
|
+
mkdir dce-project && cd dce-project
|
|
109
|
+
```
|
|
110
|
+
|
|
111
|
+
1. Initialize a new project:
|
|
112
|
+
|
|
113
|
+
```bash
|
|
114
|
+
dce init
|
|
115
|
+
```
|
|
116
|
+
|
|
117
|
+
### 2. Configure data sources
|
|
118
|
+
|
|
119
|
+
1. When prompted, agree to create a new datasource.
|
|
120
|
+
You can also use the `dce datasource add` command.
|
|
121
|
+
|
|
122
|
+
1. Provide the data source type and its name.
|
|
123
|
+
|
|
124
|
+
1. Open the config file that was created for you in your editor and fill in the connection details.
|
|
125
|
+
|
|
126
|
+
1. Repeat these steps for all data sources you want to include in your project.
|
|
127
|
+
|
|
128
|
+
1. If you have data in Markdown or text files,
|
|
129
|
+
you can add them to the `dce/src/files` directory.
|
|
130
|
+
|
|
131
|
+
### 3. Build context
|
|
132
|
+
|
|
133
|
+
1. To build the context, run the following command:
|
|
134
|
+
|
|
135
|
+
```bash
|
|
136
|
+
dce build
|
|
137
|
+
```
|
|
138
|
+
|
|
139
|
+
### 4. Use Context with Your LLM
|
|
140
|
+
|
|
141
|
+
**Option A: Dynamic via MCP Server**
|
|
142
|
+
|
|
143
|
+
Databao Context Engine exposes the context through a local MCP Server, so your agent can access the latest context at runtime.
|
|
144
|
+
|
|
145
|
+
1. In **Claude Desktop**, **Cursor**, or another MCP-compatible agent, add the following configuration.
|
|
146
|
+
Replace `dce-project/` with the path to your project directory:
|
|
147
|
+
|
|
148
|
+
```json
|
|
149
|
+
# claude_desktop_config.json, mcp.json, or similar
|
|
150
|
+
|
|
151
|
+
{
|
|
152
|
+
"mcpServers": {
|
|
153
|
+
"dce": {
|
|
154
|
+
"command": "dce mcp",
|
|
155
|
+
"args": ["--project-dir", "dce-project/"]
|
|
156
|
+
}
|
|
157
|
+
}
|
|
158
|
+
}
|
|
159
|
+
```
|
|
160
|
+
|
|
161
|
+
1. Save the file and restart your agent.
|
|
162
|
+
|
|
163
|
+
1. Open a new chat, in the chat window, select the `dce` server, and ask questions related to your project context.
|
|
164
|
+
|
|
165
|
+
**Option B: Static artifact**
|
|
166
|
+
|
|
167
|
+
Even if you don’t have Claude or Cursor installed on your local machine,
|
|
168
|
+
you can still use the context built by Databao Context Engine by pasting it directly into your chat with an AI assistant.
|
|
169
|
+
|
|
170
|
+
1. Navigate to `dce-project/output/` and open the directory with the latest run.
|
|
171
|
+
|
|
172
|
+
1. Attach the `all_results.yaml` file to your chat with the AI assistant or copy and paste its contents into your chat.
|
|
173
|
+
|
|
174
|
+
## Contributing
|
|
175
|
+
|
|
176
|
+
We’d love your help! Here’s how to get involved:
|
|
177
|
+
|
|
178
|
+
- ⭐ **Star this repo** — it helps others find us!
|
|
179
|
+
- 🐛 **Found a bug?** [Open an issue](https://github.com/JetBrains/databao-context-engine/issues)
|
|
180
|
+
- 💡 **Have an idea?** We’re all ears — create a feature request
|
|
181
|
+
- 👍 **Upvote issues** you care about — helps us prioritize
|
|
182
|
+
- 🔧 **Submit a PR**
|
|
183
|
+
- 📝 **Improve docs** — typos, examples, tutorials — everything helps!
|
|
184
|
+
|
|
185
|
+
New to open source? No worries! We're friendly and happy to help you get started. 🌱
|
|
186
|
+
|
|
187
|
+
For more details, see [CONTRIBUTING](CONTRIBUTING.md).
|
|
188
|
+
|
|
189
|
+
## 📄 License
|
|
190
|
+
|
|
191
|
+
Apache 2.0 — use it however you want. See the [LICENSE](LICENSE.md) file for details.
|
|
192
|
+
|
|
193
|
+
---
|
|
194
|
+
|
|
195
|
+
<p align="center">
|
|
196
|
+
<b>Like Databao Context Engine?</b> Give us a ⭐ — it means a lot!
|
|
197
|
+
</p>
|
|
198
|
+
|
|
199
|
+
<p align="center">
|
|
200
|
+
<a href="https://databao.app">Website</a> •
|
|
201
|
+
<a href="https://discord.gg/databao">Discord</a>
|
|
202
|
+
</p>
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "databao-context-engine"
|
|
3
|
-
version = "0.1.
|
|
4
|
-
description = "
|
|
3
|
+
version = "0.1.5"
|
|
4
|
+
description = "Semantic context for your LLMs — generated automatically"
|
|
5
5
|
readme = "README.md"
|
|
6
6
|
requires-python = ">=3.12"
|
|
7
7
|
dependencies = [
|
|
@@ -48,7 +48,7 @@ dev = [
|
|
|
48
48
|
"pre-commit>=4.5.1",
|
|
49
49
|
"pytest>=9.0.2",
|
|
50
50
|
"pytest-unordered>=0.7.0",
|
|
51
|
-
"ruff>=0.14.
|
|
51
|
+
"ruff>=0.14.4",
|
|
52
52
|
"testcontainers==4.12.0",
|
|
53
53
|
"types-pyyaml>=6.0.12.20250915",
|
|
54
54
|
"types-pymysql>=1.1.0.20250916",
|
|
@@ -66,6 +66,18 @@ override-dependencies = ["urllib3>=2.6.3", "filelock>=3.20.3", "virtualenv>=20.3
|
|
|
66
66
|
[tool.uv.build-backend]
|
|
67
67
|
source-exclude = ["tests"]
|
|
68
68
|
|
|
69
|
+
[tool.mypy]
|
|
70
|
+
#strict = true
|
|
71
|
+
show_error_codes = true
|
|
72
|
+
check_untyped_defs = false
|
|
73
|
+
disallow_untyped_defs = false ## many errors now
|
|
74
|
+
warn_redundant_casts = true
|
|
75
|
+
warn_unused_configs = true
|
|
76
|
+
warn_unused_ignores = false
|
|
77
|
+
allow_redefinition_new = true
|
|
78
|
+
local_partial_types = true
|
|
79
|
+
files = ["src", "tests"]
|
|
80
|
+
|
|
69
81
|
[[tool.mypy.overrides]]
|
|
70
82
|
module = ["requests", "requests.*"]
|
|
71
83
|
ignore_missing_imports = true
|
{databao_context_engine-0.1.3 → databao_context_engine-0.1.5}/src/databao_context_engine/__init__.py
RENAMED
|
@@ -8,6 +8,13 @@ from databao_context_engine.datasources.check_config import (
|
|
|
8
8
|
from databao_context_engine.datasources.datasource_context import DatasourceContext
|
|
9
9
|
from databao_context_engine.datasources.types import Datasource, DatasourceId
|
|
10
10
|
from databao_context_engine.init_project import init_dce_project, init_or_get_dce_project
|
|
11
|
+
from databao_context_engine.llm import (
|
|
12
|
+
OllamaError,
|
|
13
|
+
OllamaPermanentError,
|
|
14
|
+
OllamaTransientError,
|
|
15
|
+
download_ollama_models_if_needed,
|
|
16
|
+
install_ollama_if_needed,
|
|
17
|
+
)
|
|
11
18
|
from databao_context_engine.plugin_loader import DatabaoContextPluginLoader
|
|
12
19
|
from databao_context_engine.pluginlib.build_plugin import (
|
|
13
20
|
BuildDatasourcePlugin,
|
|
@@ -16,7 +23,7 @@ from databao_context_engine.pluginlib.build_plugin import (
|
|
|
16
23
|
DatasourceType,
|
|
17
24
|
)
|
|
18
25
|
from databao_context_engine.pluginlib.config import ConfigPropertyDefinition
|
|
19
|
-
from databao_context_engine.project.info import DceInfo, get_databao_context_engine_info
|
|
26
|
+
from databao_context_engine.project.info import DceInfo, DceProjectInfo, get_databao_context_engine_info
|
|
20
27
|
from databao_context_engine.project.init_project import InitErrorReason, InitProjectError
|
|
21
28
|
from databao_context_engine.services.chunk_embedding_service import ChunkEmbeddingMode
|
|
22
29
|
|
|
@@ -35,6 +42,7 @@ __all__ = [
|
|
|
35
42
|
"DatasourceType",
|
|
36
43
|
"get_databao_context_engine_info",
|
|
37
44
|
"DceInfo",
|
|
45
|
+
"DceProjectInfo",
|
|
38
46
|
"init_dce_project",
|
|
39
47
|
"init_or_get_dce_project",
|
|
40
48
|
"InitErrorReason",
|
|
@@ -44,4 +52,9 @@ __all__ = [
|
|
|
44
52
|
"BuildPlugin",
|
|
45
53
|
"BuildDatasourcePlugin",
|
|
46
54
|
"BuildFilePlugin",
|
|
55
|
+
"install_ollama_if_needed",
|
|
56
|
+
"download_ollama_models_if_needed",
|
|
57
|
+
"OllamaError",
|
|
58
|
+
"OllamaTransientError",
|
|
59
|
+
"OllamaPermanentError",
|
|
47
60
|
]
|
|
@@ -13,7 +13,7 @@ from databao_context_engine.datasources.datasource_discovery import discover_dat
|
|
|
13
13
|
from databao_context_engine.datasources.types import DatasourceId
|
|
14
14
|
from databao_context_engine.pluginlib.build_plugin import DatasourceType
|
|
15
15
|
from databao_context_engine.plugins.plugin_loader import load_plugins
|
|
16
|
-
from databao_context_engine.project.layout import
|
|
16
|
+
from databao_context_engine.project.layout import ProjectLayout
|
|
17
17
|
|
|
18
18
|
logger = logging.getLogger(__name__)
|
|
19
19
|
|
|
@@ -36,7 +36,7 @@ class BuildContextResult:
|
|
|
36
36
|
|
|
37
37
|
|
|
38
38
|
def build(
|
|
39
|
-
|
|
39
|
+
project_layout: ProjectLayout,
|
|
40
40
|
*,
|
|
41
41
|
build_service: BuildService,
|
|
42
42
|
) -> list[BuildContextResult]:
|
|
@@ -55,15 +55,15 @@ def build(
|
|
|
55
55
|
"""
|
|
56
56
|
plugins = load_plugins()
|
|
57
57
|
|
|
58
|
-
datasources = discover_datasources(
|
|
58
|
+
datasources = discover_datasources(project_layout)
|
|
59
59
|
|
|
60
60
|
if not datasources:
|
|
61
|
-
logger.info("No sources discovered under %s",
|
|
61
|
+
logger.info("No sources discovered under %s", project_layout.src_dir)
|
|
62
62
|
return []
|
|
63
63
|
|
|
64
64
|
number_of_failed_builds = 0
|
|
65
65
|
build_result = []
|
|
66
|
-
reset_all_results(
|
|
66
|
+
reset_all_results(project_layout.output_dir)
|
|
67
67
|
for discovered_datasource in datasources:
|
|
68
68
|
try:
|
|
69
69
|
prepared_source = prepare_source(discovered_datasource)
|
|
@@ -87,14 +87,14 @@ def build(
|
|
|
87
87
|
plugin=plugin,
|
|
88
88
|
)
|
|
89
89
|
|
|
90
|
-
output_dir =
|
|
90
|
+
output_dir = project_layout.output_dir
|
|
91
91
|
|
|
92
92
|
context_file_path = export_build_result(output_dir, result)
|
|
93
93
|
append_result_to_all_results(output_dir, result)
|
|
94
94
|
|
|
95
95
|
build_result.append(
|
|
96
96
|
BuildContextResult(
|
|
97
|
-
datasource_id=
|
|
97
|
+
datasource_id=discovered_datasource.datasource_id,
|
|
98
98
|
datasource_type=DatasourceType(full_type=result.datasource_type),
|
|
99
99
|
context_built_at=result.context_built_at,
|
|
100
100
|
context_file_path=context_file_path,
|
|
@@ -1,5 +1,4 @@
|
|
|
1
1
|
import logging
|
|
2
|
-
from pathlib import Path
|
|
3
2
|
|
|
4
3
|
from duckdb import DuckDBPyConnection
|
|
5
4
|
|
|
@@ -12,7 +11,7 @@ from databao_context_engine.llm.factory import (
|
|
|
12
11
|
create_ollama_embedding_provider,
|
|
13
12
|
create_ollama_service,
|
|
14
13
|
)
|
|
15
|
-
from databao_context_engine.project.layout import
|
|
14
|
+
from databao_context_engine.project.layout import ProjectLayout
|
|
16
15
|
from databao_context_engine.services.chunk_embedding_service import ChunkEmbeddingMode
|
|
17
16
|
from databao_context_engine.services.factories import create_chunk_embedding_service
|
|
18
17
|
from databao_context_engine.storage.connection import open_duckdb_connection
|
|
@@ -22,7 +21,9 @@ from databao_context_engine.system.properties import get_db_path
|
|
|
22
21
|
logger = logging.getLogger(__name__)
|
|
23
22
|
|
|
24
23
|
|
|
25
|
-
def build_all_datasources(
|
|
24
|
+
def build_all_datasources(
|
|
25
|
+
project_layout: ProjectLayout, chunk_embedding_mode: ChunkEmbeddingMode
|
|
26
|
+
) -> list[BuildContextResult]:
|
|
26
27
|
"""Build the context for all datasources in the project.
|
|
27
28
|
|
|
28
29
|
- Instantiates the build service
|
|
@@ -31,21 +32,18 @@ def build_all_datasources(project_dir: Path, chunk_embedding_mode: ChunkEmbeddin
|
|
|
31
32
|
Returns:
|
|
32
33
|
A list of all the contexts built.
|
|
33
34
|
"""
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
logger.debug(f"Starting to build datasources in project {project_dir.resolve()}")
|
|
35
|
+
logger.debug(f"Starting to build datasources in project {project_layout.project_dir.resolve()}")
|
|
37
36
|
|
|
38
37
|
# Think about alternative solutions. This solution will mirror the current behaviour
|
|
39
38
|
# The current behaviour only builds what is currently in the /src folder
|
|
40
39
|
# This will need to change in the future when we can pick which datasources to build
|
|
41
|
-
db_path = get_db_path(project_dir)
|
|
40
|
+
db_path = get_db_path(project_layout.project_dir)
|
|
42
41
|
db_path.parent.mkdir(parents=True, exist_ok=True)
|
|
43
42
|
if db_path.exists():
|
|
44
43
|
db_path.unlink()
|
|
45
44
|
|
|
45
|
+
migrate(db_path)
|
|
46
46
|
with open_duckdb_connection(db_path) as conn:
|
|
47
|
-
migrate(db_path)
|
|
48
|
-
|
|
49
47
|
ollama_service = create_ollama_service()
|
|
50
48
|
embedding_provider = create_ollama_embedding_provider(ollama_service)
|
|
51
49
|
description_provider = (
|
|
@@ -60,7 +58,7 @@ def build_all_datasources(project_dir: Path, chunk_embedding_mode: ChunkEmbeddin
|
|
|
60
58
|
chunk_embedding_mode=chunk_embedding_mode,
|
|
61
59
|
)
|
|
62
60
|
return build(
|
|
63
|
-
|
|
61
|
+
project_layout=project_layout,
|
|
64
62
|
build_service=build_service,
|
|
65
63
|
)
|
|
66
64
|
|
|
@@ -2,13 +2,13 @@ from dataclasses import dataclass
|
|
|
2
2
|
from datetime import datetime
|
|
3
3
|
from typing import Any, cast
|
|
4
4
|
|
|
5
|
+
from databao_context_engine.datasources.types import PreparedConfig, PreparedDatasource
|
|
5
6
|
from databao_context_engine.pluginlib.build_plugin import (
|
|
6
7
|
BuildDatasourcePlugin,
|
|
7
8
|
BuildFilePlugin,
|
|
8
9
|
BuildPlugin,
|
|
9
10
|
)
|
|
10
11
|
from databao_context_engine.pluginlib.plugin_utils import execute_datasource_plugin, execute_file_plugin
|
|
11
|
-
from databao_context_engine.datasources.types import PreparedConfig, PreparedDatasource, DatasourceId
|
|
12
12
|
|
|
13
13
|
|
|
14
14
|
@dataclass()
|
|
@@ -39,11 +39,8 @@ class BuiltDatasourceContext:
|
|
|
39
39
|
|
|
40
40
|
def execute(prepared_datasource: PreparedDatasource, plugin: BuildPlugin) -> BuiltDatasourceContext:
|
|
41
41
|
built_context = _execute(prepared_datasource, plugin)
|
|
42
|
-
|
|
43
|
-
datasource_id = DatasourceId.from_datasource_config_file_path(prepared_datasource.path)
|
|
44
|
-
|
|
45
42
|
return BuiltDatasourceContext(
|
|
46
|
-
datasource_id=str(datasource_id),
|
|
43
|
+
datasource_id=str(prepared_datasource.datasource_id),
|
|
47
44
|
datasource_type=prepared_datasource.datasource_type.full_type,
|
|
48
45
|
context_built_at=datetime.now(),
|
|
49
46
|
context=built_context,
|
|
@@ -61,10 +58,10 @@ def _execute(prepared_datasource: PreparedDatasource, plugin: BuildPlugin) -> An
|
|
|
61
58
|
config=prepared_datasource.config,
|
|
62
59
|
datasource_name=prepared_datasource.datasource_name,
|
|
63
60
|
)
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
61
|
+
|
|
62
|
+
file_plugin = cast(BuildFilePlugin, plugin)
|
|
63
|
+
return execute_file_plugin(
|
|
64
|
+
plugin=file_plugin,
|
|
65
|
+
datasource_type=prepared_datasource.datasource_type,
|
|
66
|
+
file_path=prepared_datasource.path,
|
|
67
|
+
)
|