databao-context-engine 0.1.2__py3-none-any.whl → 0.1.4.dev1__py3-none-any.whl

databao_context_engine-0.1.4.dev1.dist-info/RECORD

@@ -0,0 +1,125 @@
+databao_context_engine/__init__.py,sha256=nYkaM5t5Y5EV-p9_Ajg62xLFHW24liMc79oKxyodGYc,1874
+databao_context_engine/build_sources/__init__.py,sha256=FteGp3MvSuX_fh-Fx1pq-jOjFcjYKQSXFSll4tQffvk,224
+databao_context_engine/build_sources/build_runner.py,sha256=YXU-EJoJ5nzGePrvWA8nxJl-k-PKwgShiKPHiN5ndLM,3981
+databao_context_engine/build_sources/build_service.py,sha256=eywJzpvZUVrj-JYxXii_kxhTBzVUT0xfbNtdAFcpGWQ,1624
+databao_context_engine/build_sources/build_wiring.py,sha256=7Jw78gCy57Zy0e5nUnAqQmPdTkW6tztlHnQmRJ1cNkE,3192
+databao_context_engine/build_sources/export_results.py,sha256=E3HP4uaQKYDrmAJKFWwDT190j7ZKfO791_pjKebTMQM,1773
+databao_context_engine/build_sources/plugin_execution.py,sha256=Icl2oJeyL98aD8m0unmnRdtCOtaZE6iRGtmCUFA_eYk,2211
+databao_context_engine/cli/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+databao_context_engine/cli/add_datasource_config.py,sha256=ESAIkLb_g2Bg-WjorZ6JFSwxh23SY-g1T699_fxip54,6371
+databao_context_engine/cli/commands.py,sha256=ntqoThbplgrHPtDy5kKna1Cz_iu8UKJQG6q7G6UmsCs,7106
+databao_context_engine/cli/datasources.py,sha256=kHbXhEnDVaYXTRAE3WeaDGohH1Y9cwyYjjcycQ8eh1U,2376
+databao_context_engine/cli/info.py,sha256=2iegfvRAxMae3KkWMDu-BOnljeJMVQ5u8P5k_8nOKpc,1203
+databao_context_engine/config/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+databao_context_engine/config/log_config.yaml,sha256=hH2NPumufcyig8L-7Z_DbLQ88_G2N6CAEl5fvHafeAw,360
+databao_context_engine/config/logging.py,sha256=66k7x3ftEdHx-CKqbOss4pqZfGn4nMFv7AzVPs9aVnY,1525
+databao_context_engine/databao_context_engine.py,sha256=_45hvUt9u0CDhhoZDjByf12jw_QjQgH_fa0DM1D2zTc,5416
+databao_context_engine/databao_context_project_manager.py,sha256=NVkVEpW_82iQuBTnEoJDNFswidR6nzTP9h6m0NDCogg,7502
+databao_context_engine/datasources/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+databao_context_engine/datasources/add_config.py,sha256=NN6wenXWAYxC_f65HRh8NZRMO86q3tDxi5Y2mX9qiR4,1208
+databao_context_engine/datasources/check_config.py,sha256=DefL4SkrdlWVjboN5KgZ4lMtqPC7Y5hTnvNtPhkMmE4,5475
+databao_context_engine/datasources/datasource_context.py,sha256=fK1MEvlDwPvybqIQ_Xkw2SyZd_p2X0gPUP9hX6AfDhU,3542
+databao_context_engine/datasources/datasource_discovery.py,sha256=SbqYdqg59M9f_a3TKAf4uXNm2hCDJ-U28Jw4cE-8h5g,5198
+databao_context_engine/datasources/types.py,sha256=KO4AiYvh7qbMyyPmzhZDuByCHkiFSOe_SvnwrYSiw0Q,7063
+databao_context_engine/event_journal/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+databao_context_engine/event_journal/writer.py,sha256=abkl2SPHYzO8XAUsjbtrcXm6WQk15zWpXhYm86qb6G0,986
+databao_context_engine/generate_configs_schemas.py,sha256=OBX1Motg_d-fmZCUnDoF-3c8dYbDcTOJmhD7F0WJHH8,3017
+databao_context_engine/init_project.py,sha256=enLvYOd5MpGYVdWly7DuLMzPLUEE6130bEuxh0YZft8,1496
+databao_context_engine/introspection/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+databao_context_engine/introspection/property_extract.py,sha256=m7DL8SWNIipi8uE9q4k3TDvlga9YeKyKh85CvFt_fqI,7816
+databao_context_engine/llm/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+databao_context_engine/llm/config.py,sha256=geIygR-9maeKbvmcO5J3iNZXKS4LZc7zpKYyaH8vCoI,463
+databao_context_engine/llm/descriptions/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+databao_context_engine/llm/descriptions/ollama.py,sha256=XhXQD8j7OAHiKL0TjZPpNtDDoIV0NI0u92PTksejc5g,657
+databao_context_engine/llm/descriptions/provider.py,sha256=nojEI0l55TfFuNooQwGNvfa2k6A1vZ1Oi3PBLAr9eFo,227
+databao_context_engine/llm/embeddings/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+databao_context_engine/llm/embeddings/ollama.py,sha256=GQzlMBRJGJe_xreY9-Y3REBFWWaOyKhiGQxWxrR6xfY,949
+databao_context_engine/llm/embeddings/provider.py,sha256=IVrOptwad2Fr1wf1SBOM3-7y66k08mOZnga1NA6yhbA,300
+databao_context_engine/llm/errors.py,sha256=Y4CVHYJKkn2oHnFWVlVL1QJl4gtvW4z11V3FB9qp3Cc,402
+databao_context_engine/llm/factory.py,sha256=VSCVAWXV6d496SWDyUH4rGYGXQ3Q6yg9Ymc8C6x8EXQ,1765
+databao_context_engine/llm/install.py,sha256=8_rtcr-o03FyW7nieX7xE24oVhxVa_nrd97CC0hbbZs,6555
+databao_context_engine/llm/runtime.py,sha256=jf_sPDF4s_fChFj2enLcEIBXIWjSjaYNh-l6lXIgRd4,2173
+databao_context_engine/llm/service.py,sha256=rM_DrF2F7NylxBhtfVGporUcv91eR_nF1mF--Zxh2tc,5978
+databao_context_engine/main.py,sha256=3gWwEc9GZJUFANJtW9Fx4smVzib_sDtFkBGPILjisaI,350
+databao_context_engine/mcp/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+databao_context_engine/mcp/mcp_runner.py,sha256=bnB2a-_P1zOlDBqpyFIq4n2hWkfwqb6G7dHb3-3Q2I0,468
+databao_context_engine/mcp/mcp_server.py,sha256=fXkl-LSXjWQwYoranWgBUh5qrIcTv3ZOJvKfbnf0HwM,2231
+databao_context_engine/plugin_loader.py,sha256=2vq7rCNQuxQvnGqF758Ke2IzCO-11dLD2lH8BKP1b68,4829
+databao_context_engine/pluginlib/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+databao_context_engine/pluginlib/build_plugin.py,sha256=v90JU4JRqYb4BbMvx833lPVafBUOE7Bhpt8SOWdBrZU,4105
+databao_context_engine/pluginlib/config.py,sha256=fAu1AuxSFI6LOoAyvwuFE5p11bOsHp9EZzn2ST8DrUI,1534
+databao_context_engine/pluginlib/plugin_utils.py,sha256=f76ksdMUG3Wy4KOytUYxLMrq9iOHytu3QF8wc5NzzJE,2471
+databao_context_engine/plugins/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+databao_context_engine/plugins/athena_db_plugin.py,sha256=53Dui_V7MunJ3jmyYEAmkZofuHtQP-Ic3bNhsxTwAyc,456
+databao_context_engine/plugins/base_db_plugin.py,sha256=NkDR7aHqskoBwtOJbfiMdn_nWNe94lbq_ZbciiDbfiI,1733
+databao_context_engine/plugins/clickhouse_db_plugin.py,sha256=4J1CNoeWPRvgmgkww7CQALxTP5D9eSLl1TpDRgwHBew,509
+databao_context_engine/plugins/databases/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+databao_context_engine/plugins/databases/athena_introspector.py,sha256=--F8sbtPdcCUtDfMzx85_zH0sBlArJigM6Aa1mmn00w,5702
+databao_context_engine/plugins/databases/base_introspector.py,sha256=aXMWJ13LdYs1pSaGIhtA8zefe-oXlmzM6X_Ur8ZbMok,5760
+databao_context_engine/plugins/databases/clickhouse_introspector.py,sha256=6EyV46QR9UhcClnmBNxRzxgShvedbOo58p77hPFcK2s,6353
+databao_context_engine/plugins/databases/database_chunker.py,sha256=dn9Mb9Sw4f8IwBe9RtukQGT0yyKLQt5i4tjvsrFh1ew,2100
+databao_context_engine/plugins/databases/databases_types.py,sha256=kYZEaFa_cnROg_n1k7tIonoC2VunLBe5SQR47GOt_8k,2579
+databao_context_engine/plugins/databases/duckdb_introspector.py,sha256=temeOVxUY46vOgHZ9g-ra0n5thCEthvGTD7g3dZJkBk,11332
+databao_context_engine/plugins/databases/introspection_model_builder.py,sha256=-i7e4jqwdcsu3NnS-LHVSH_DrG3riN09KX3SDHAGF0c,10350
+databao_context_engine/plugins/databases/introspection_scope.py,sha256=0BZAAmVTiHifOzeuQkCsGKrGRnZCPHRCLqam5nQqr88,2225
+databao_context_engine/plugins/databases/introspection_scope_matcher.py,sha256=gseF4j-SD-7SSPMDlDqHUOTI3tPSaOGKJgp42WDVlsY,3605
+databao_context_engine/plugins/databases/mssql_introspector.py,sha256=mw6z3YUKQ5hzFFWpNhx5NYrOG6tfQIclWESvdcUhIoY,18793
+databao_context_engine/plugins/databases/mysql_introspector.py,sha256=Y77snyZkTT5elL7bsbzjUeffugLl_v1NB4yMQFRbCso,14504
+databao_context_engine/plugins/databases/postgresql_introspector.py,sha256=tHsQNWJLHcH4GUYwmXSf5mY6XIMhvtpUbGgwmWfljpw,17489
+databao_context_engine/plugins/databases/snowflake_introspector.py,sha256=K3v25O-jMybM9Iey5Wqub5oGLjcHsSaKDu4VXWMzu70,12005
+databao_context_engine/plugins/duckdb_db_plugin.py,sha256=uumf9kXz6U-6QbX0i1ChHHxxvdq5_di3gTNpVDUCYdg,451
+databao_context_engine/plugins/duckdb_tools.py,sha256=46rctnTxDPAhHtaiTp1DxMuuDuRKrtKWJFSSM2w7uUU,645
+databao_context_engine/plugins/mssql_db_plugin.py,sha256=YVor2QMDuL6XOLT8RFPOgs1Uhj1ZVYoFP5DApDJFPaE,446
+databao_context_engine/plugins/mysql_db_plugin.py,sha256=wy8HTiYuE8GTxfoc5gdrinEYv7NWHGrslzaAzruQecY,446
+databao_context_engine/plugins/parquet_plugin.py,sha256=qlpzQmiEwpb8Jma50HoIvljGV5ZiWkZU7GSiGV7M6kc,1181
+databao_context_engine/plugins/plugin_loader.py,sha256=Nx_OYGf9t3XKb_JV4Om9fCOuL7B-qVw-peXKXHiK8G0,3861
+databao_context_engine/plugins/postgresql_db_plugin.py,sha256=PwLErcbqixsgz4Tqoift0uyerUqP_OaRzMaMAn0TE5A,486
+databao_context_engine/plugins/resources/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+databao_context_engine/plugins/resources/parquet_chunker.py,sha256=R9WCOBqpKRTVN6t5eeOm_mmnKBOxvjIiQ9zTc8vnUb4,848
+databao_context_engine/plugins/resources/parquet_introspector.py,sha256=dnf_rLQYd9RWIevl64_8ddhXO5AFQhcuOWQiaRCOeYM,5639
+databao_context_engine/plugins/snowflake_db_plugin.py,sha256=ApiVE212DU9F4Zj1HPof6JQ66n4LUeYRRwhlTWRCxek,486
+databao_context_engine/plugins/unstructured_files_plugin.py,sha256=TWV3fluusizul2YEN-YZ-K-C3s5W-5B8353-bjP53L8,2424
+databao_context_engine/project/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+databao_context_engine/project/info.py,sha256=3chVSZ4pV2EsYJSeVfOYTK746POlJImciIBOvrQuYbc,2479
+databao_context_engine/project/init_project.py,sha256=QiLalH--BMQGB-c2WBbPAUKBinDbO0cJkmilUfqPAuI,3970
+databao_context_engine/project/layout.py,sha256=7ThzqnFejMogBV8c6Mt-Md4GO-kdzYNrmhhBTalYvps,4295
+databao_context_engine/project/project_config.py,sha256=rI-Wkll7lca7RlYIaFScs5CIKIZ8uujtKRxSjibGIt8,1132
+databao_context_engine/project/resources/examples/src/databases/example_postgres.yaml,sha256=5o6jF_zR4WSvS9QcZnGW-S2FmahsjNp-tMbAkzbNabo,139
+databao_context_engine/project/resources/examples/src/files/documentation.md,sha256=k5G5lrN4S_ie2o7CBxXgLiWGIGtb3v-77c_3n4C7Lvw,1090
+databao_context_engine/project/resources/examples/src/files/notes.txt,sha256=0T1u8lIwIRwNopEsfWmqfIbhfRl-DN3XuiIN7suRAiE,3668
+databao_context_engine/retrieve_embeddings/__init__.py,sha256=__aOhCc3sBfLtjxz0hlVGuSYUMRsEM3PucPYNC6LVq0,126
+databao_context_engine/retrieve_embeddings/export_results.py,sha256=_IdhKPQQhK9o0uCLwRqoNTvStS3qh8UhdICfEADkFec,337
+databao_context_engine/retrieve_embeddings/retrieve_runner.py,sha256=-nvsGssxEQkXoL4D02L_RDYGuvbsN0icpFmw6JF4W-s,1046
+databao_context_engine/retrieve_embeddings/retrieve_service.py,sha256=KrCNEu-Yd7dZrk0jk9ZOp4yO5WNjuzg7l4CgYLMqFvI,2005
+databao_context_engine/retrieve_embeddings/retrieve_wiring.py,sha256=nO_OTrbKOsvgJz_CiqZliIdCg1L3m6cWELyTI2nVZgk,2123
+databao_context_engine/serialization/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+databao_context_engine/serialization/yaml.py,sha256=e0xQBgDmrCspppAVyonMkVkznOjuUcOGjfNEi8Gmc74,1255
+databao_context_engine/services/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+databao_context_engine/services/chunk_embedding_service.py,sha256=Dy25u2QSSdq03v1zs2bjjm8Sze-VEd12F49gzoKXjOA,4924
+databao_context_engine/services/embedding_shard_resolver.py,sha256=6oOVm6QJb74lEIbYXijWMqISTx1VfmKQaQjdSHr8zDA,2289
+databao_context_engine/services/factories.py,sha256=IoRmJk9a7q2eszwz_zfqE2hJDOwpB76SmmmgR0DI-qY,1830
+databao_context_engine/services/models.py,sha256=wHmk4eheoSopXkCs3v2ggoDAGoTPdHYcADMZ4gdtSCI,299
+databao_context_engine/services/persistence_service.py,sha256=Kcu4rw-TCqyHiGW1h5rYZfe3F-HZTn0tRAM4_czzEAc,2252
+databao_context_engine/services/table_name_policy.py,sha256=q0scAQ_ZoTrV8V0J6cmxvsMYxTcmUlAqEWpzZUitpLQ,567
+databao_context_engine/storage/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+databao_context_engine/storage/connection.py,sha256=UGnJOBJ83jDAmN3fmjVZI60qUw5cAC3YNLToJZLOhGQ,941
+databao_context_engine/storage/exceptions/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+databao_context_engine/storage/exceptions/exceptions.py,sha256=zwJ8n-ekMhIH-bs9dvYqQuP4VfQ9rt0jdsptoojA5oU,174
+databao_context_engine/storage/migrate.py,sha256=yGt9VdSsR0g3ZxesTCWs4eyJHk9EPIuJakeRpCFWEyY,4594
+databao_context_engine/storage/migrations/V01__init.sql,sha256=hJvUoPIW-rQQB5DSDrPjGJQoey9bpDRz0FFoUyTe9Qo,1368
+databao_context_engine/storage/models.py,sha256=TmoeyOhrFL9mu-C43XUtqOWGhkQu2h-sZQdYCjui5HM,580
+databao_context_engine/storage/repositories/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+databao_context_engine/storage/repositories/chunk_repository.py,sha256=nwDF8NeCwQJLlwtlvqILUccmWmQTfpg6Uf_WCDXm564,3681
+databao_context_engine/storage/repositories/embedding_model_registry_repository.py,sha256=g91WCflYRc5jPDbW9RBp5HxWFsSIta4n8OYNwgoLv9c,2413
+databao_context_engine/storage/repositories/embedding_repository.py,sha256=UCvr5mf2jTJgX_uMdLIMiN5NWa8Zn1U3HEr19TQRXAw,3278
+databao_context_engine/storage/repositories/factories.py,sha256=YUkVrHyf5UV2rZ7H97Fi_n0ZYxTwXy8_w5zJnMkO8Lw,970
+databao_context_engine/storage/repositories/vector_search_repository.py,sha256=D6gebioSvqWslJw5zSleEG1Inj2w8MkPfbgGZFTZDfs,1944
+databao_context_engine/storage/transaction.py,sha256=QpfE1VtgDd3_7OBzFYI3HPJJ7ESXkgCdcWukFkAjbSg,270
+databao_context_engine/system/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+databao_context_engine/system/properties.py,sha256=mQ7-_PZeYSESYn1cMUQ0IK7rJEnbhc7t4WesFjAgo-Q,429
+databao_context_engine/templating/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+databao_context_engine/templating/renderer.py,sha256=W2-0IGStAp6oxANmsKs_Z-UoIR6Gt_c4ILYFa3Hruo4,662
+databao_context_engine-0.1.4.dev1.dist-info/WHEEL,sha256=5w2T7AS2mz1-rW9CNagNYWRCaB0iQqBMYLwKdlgiR4Q,78
+databao_context_engine-0.1.4.dev1.dist-info/entry_points.txt,sha256=5EeQJ1W8zEFh4HuF1bs2zBeoP408oiwuM9UrkJiurgI,138
+databao_context_engine-0.1.4.dev1.dist-info/METADATA,sha256=43INtHVdc-AdDCHzRA-bHxGP66x_-kaXbKi1ZSapp3M,3367
+databao_context_engine-0.1.4.dev1.dist-info/RECORD,,

{databao_context_engine-0.1.2.dist-info → databao_context_engine-0.1.4.dev1.dist-info}/WHEEL

@@ -1,4 +1,4 @@
 Wheel-Version: 1.0
-Generator: uv 0.9.24
+Generator: uv 0.9.7
 Root-Is-Purelib: true
 Tag: py3-none-any

databao_context_engine/build_sources/internal/build_service.py

@@ -1,77 +0,0 @@
-from __future__ import annotations
-
-import logging
-from datetime import datetime
-
-from databao_context_engine.build_sources.internal.plugin_execution import BuiltDatasourceContext, execute
-from databao_context_engine.pluginlib.build_plugin import (
-    BuildPlugin,
-)
-from databao_context_engine.project.types import PreparedDatasource
-from databao_context_engine.serialisation.yaml import to_yaml_string
-from databao_context_engine.services.chunk_embedding_service import ChunkEmbeddingService
-from databao_context_engine.storage.models import RunDTO
-from databao_context_engine.storage.repositories.datasource_run_repository import DatasourceRunRepository
-from databao_context_engine.storage.repositories.run_repository import RunRepository
-
-logger = logging.getLogger(__name__)
-
-
-class BuildService:
-    def __init__(
-        self,
-        *,
-        run_repo: RunRepository,
-        datasource_run_repo: DatasourceRunRepository,
-        chunk_embedding_service: ChunkEmbeddingService,
-    ) -> None:
-        self._run_repo = run_repo
-        self._datasource_run_repo = datasource_run_repo
-        self._chunk_embedding_service = chunk_embedding_service
-
-    def start_run(self, *, project_id: str, dce_version: str) -> RunDTO:
-        """
-        Create a new run row and return (run_id, started_at).
-        """
-        return self._run_repo.create(project_id=project_id, dce_version=dce_version)
-
-    def finalize_run(self, *, run_id: int):
-        """
-        Mark the run as complete (sets ended_at).
-        """
-        self._run_repo.update(run_id=run_id, ended_at=datetime.now())
-
-    def process_prepared_source(
-        self,
-        *,
-        run_id: int,
-        prepared_source: PreparedDatasource,
-        plugin: BuildPlugin,
-    ) -> BuiltDatasourceContext:
-        """
-        Process a single source.
-
-        1) Execute the plugin
-        2) Divide the results into chunks
-        3) Embed and persist the chunks
-        """
-        result = execute(prepared_source, plugin)
-
-        chunks = plugin.divide_context_into_chunks(result.context)
-        if not chunks:
-            logger.info("No chunks for %s — skipping.", prepared_source.path.name)
-            return result
-
-        datasource_run = self._datasource_run_repo.create(
-            run_id=run_id,
-            plugin=plugin.name,
-            full_type=prepared_source.datasource_type.full_type,
-            source_id=result.datasource_id,
-            storage_directory=str(prepared_source.path.parent),
-        )
-
-        self._chunk_embedding_service.embed_chunks(
-            datasource_run_id=datasource_run.datasource_run_id, chunks=chunks, result=to_yaml_string(result.context)
-        )
-
-        return result

databao_context_engine/build_sources/internal/build_wiring.py

@@ -1,52 +0,0 @@
-import logging
-from pathlib import Path
-
-from databao_context_engine.build_sources.internal.build_runner import BuildContextResult, build
-from databao_context_engine.llm.factory import (
-    create_ollama_description_provider,
-    create_ollama_embedding_provider,
-    create_ollama_service,
-)
-from databao_context_engine.project.info import get_dce_version
-from databao_context_engine.project.layout import ensure_project_dir
-from databao_context_engine.services.chunk_embedding_service import ChunkEmbeddingMode
-from databao_context_engine.services.factories import (
-    create_build_service,
-)
-from databao_context_engine.storage.connection import open_duckdb_connection
-from databao_context_engine.system.properties import get_db_path
-
-logger = logging.getLogger(__name__)
-
-
-def build_all_datasources(project_dir: Path, chunk_embedding_mode: ChunkEmbeddingMode) -> list[BuildContextResult]:
-    """
-    Public build entrypoint
-    - Instantiates the build service
-    - Delegates the actual build logic to the build runner
-    """
-    project_layout = ensure_project_dir(project_dir)
-
-    logger.debug(f"Starting to build datasources in project {project_dir.resolve()}")
-
-    with open_duckdb_connection(get_db_path()) as conn:
-        ollama_service = create_ollama_service()
-        embedding_provider = create_ollama_embedding_provider(ollama_service)
-        description_provider = (
-            create_ollama_description_provider(ollama_service)
-            if chunk_embedding_mode.should_generate_description()
-            else None
-        )
-        build_service = create_build_service(
-            conn,
-            embedding_provider=embedding_provider,
-            description_provider=description_provider,
-            chunk_embedding_mode=chunk_embedding_mode,
-        )
-        dce_config = project_layout.read_config_file()
-        return build(
-            project_dir=project_dir,
-            build_service=build_service,
-            project_id=str(dce_config.project_id),
-            dce_version=get_dce_version(),
-        )

databao_context_engine/build_sources/internal/export_results.py

@@ -1,43 +0,0 @@
-import logging
-from pathlib import Path
-
-from databao_context_engine.build_sources.internal.plugin_execution import BuiltDatasourceContext
-from databao_context_engine.datasource_config.datasource_context import get_context_header_for_datasource
-from databao_context_engine.project.layout import ALL_RESULTS_FILE_NAME, get_output_dir
-from databao_context_engine.project.types import DatasourceId
-from databao_context_engine.serialisation.yaml import write_yaml_to_stream
-
-logger = logging.getLogger(__name__)
-
-
-def create_run_dir(project_dir: Path, run_name: str) -> Path:
-    output_dir = get_output_dir(project_dir)
-
-    run_dir = output_dir.joinpath(run_name)
-    run_dir.mkdir(parents=True, exist_ok=False)
-
-    return run_dir
-
-
-def export_build_result(run_dir: Path, result: BuiltDatasourceContext) -> Path:
-    datasource_id = DatasourceId.from_string_repr(result.datasource_id)
-    export_file_path = run_dir.joinpath(datasource_id.relative_path_to_context_file())
-
-    # Make sure the parent folder exists
-    export_file_path.parent.mkdir(exist_ok=True)
-
-    with export_file_path.open("w") as export_file:
-        write_yaml_to_stream(data=result, file_stream=export_file)
-
-    logger.info(f"Exported result to {export_file_path.resolve()}")
-
-    return export_file_path
-
-
-def append_result_to_all_results(run_dir: Path, result: BuiltDatasourceContext):
-    path = run_dir.joinpath(ALL_RESULTS_FILE_NAME)
-    path.parent.mkdir(parents=True, exist_ok=True)
-    with path.open("a", encoding="utf-8") as export_file:
-        export_file.write(get_context_header_for_datasource(DatasourceId.from_string_repr(result.datasource_id)))
-        write_yaml_to_stream(data=result, file_stream=export_file)
-        export_file.write("\n")

databao_context_engine/build_sources/public/api.py

@@ -1,4 +0,0 @@
-from databao_context_engine.build_sources.internal.build_runner import BuildContextResult
-from databao_context_engine.build_sources.internal.build_wiring import build_all_datasources
-
-__all__ = ["build_all_datasources", "BuildContextResult"]

databao_context_engine/databao_engine.py

@@ -1,85 +0,0 @@
-import os
-from dataclasses import dataclass
-from pathlib import Path
-from typing import Any
-
-from databao_context_engine.datasource_config.datasource_context import (
-    DatasourceContext,
-    get_all_contexts,
-    get_context_header_for_datasource,
-    get_datasource_context,
-)
-from databao_context_engine.pluginlib.build_plugin import DatasourceType
-from databao_context_engine.project.datasource_discovery import get_datasource_list
-from databao_context_engine.project.layout import ensure_project_dir
-from databao_context_engine.project.types import Datasource, DatasourceId
-from databao_context_engine.retrieve_embeddings.public.api import retrieve_embeddings
-
-
-@dataclass
-class ContextSearchResult:
-    datasource_id: DatasourceId
-    datasource_type: DatasourceType
-    distance: float
-    context_result: str
-
-
-class DatabaoContextEngine:
-    project_dir: Path
-
-    def __init__(self, project_dir: Path) -> None:
-        self.project_layout = ensure_project_dir(project_dir=project_dir)
-        self.project_dir = project_dir
-
-    def get_datasource_list(self) -> list[Datasource]:
-        # TODO: Should this return the list of built datasources rather than the list of datasources within the src folder?
-        return get_datasource_list(self.project_dir)
-
-    def get_datasource_context(self, datasource_id: DatasourceId, run_name: str | None = None) -> DatasourceContext:
-        return get_datasource_context(
-            project_layout=self.project_layout, datasource_id=datasource_id, run_name=run_name
-        )
-
-    def get_all_contexts(self, run_name: str | None = None) -> list[DatasourceContext]:
-        return get_all_contexts(project_layout=self.project_layout, run_name=run_name)
-
-    def get_all_contexts_formatted(self, run_name: str | None = None) -> str:
-        all_contexts = self.get_all_contexts(run_name=run_name)
-
-        all_results = os.linesep.join(
-            [f"{get_context_header_for_datasource(context.datasource_id)}{context.context}" for context in all_contexts]
-        )
-
-        return all_results
-
-    def search_context(
-        self,
-        retrieve_text: str,
-        run_name: str | None,
-        limit: int | None,
-        export_to_file: bool,
-        datasource_ids: list[DatasourceId] | None = None,
-    ) -> list[ContextSearchResult]:
-        # TODO: Filter with datasource_ids
-        # TODO: Remove the need for a run_name
-
-        results = retrieve_embeddings(
-            project_layout=self.project_layout,
-            retrieve_text=retrieve_text,
-            run_name=run_name,
-            limit=limit,
-            export_to_file=export_to_file,
-        )
-
-        return [
-            ContextSearchResult(
-                datasource_id=result.datasource_id,
-                datasource_type=result.datasource_type,
-                distance=result.cosine_distance,
-                context_result=result.display_text,
-            )
-            for result in results
-        ]
-
-    def run_sql(self, datasource_id: DatasourceId, sql: str, params: list[str]) -> dict[str, Any]:
-        raise NotImplementedError("Running SQL is not supported yet")

databao_context_engine/datasource_config/add_config.py

@@ -1,50 +0,0 @@
-from pathlib import Path
-from typing import Any
-
-from databao_context_engine.introspection.property_extract import get_property_list_from_type
-from databao_context_engine.pluginlib.build_plugin import BuildDatasourcePlugin, DatasourceType
-from databao_context_engine.pluginlib.config import ConfigPropertyDefinition, CustomiseConfigProperties
-from databao_context_engine.plugins.plugin_loader import get_plugin_for_type
-from databao_context_engine.project.layout import (
-    create_datasource_config_file as create_datasource_config_file_internal,
-)
-from databao_context_engine.project.types import DatasourceId
-from databao_context_engine.serialisation.yaml import to_yaml_string
-
-
-def get_config_file_structure_for_datasource_type(datasource_type: DatasourceType) -> list[ConfigPropertyDefinition]:
-    plugin = get_plugin_for_type(datasource_type)
-
-    if isinstance(plugin, CustomiseConfigProperties):
-        return plugin.get_config_file_properties()
-    elif isinstance(plugin, BuildDatasourcePlugin):
-        return get_property_list_from_type(plugin.config_file_type)
-    else:
-        raise ValueError(
-            f"Impossible to create a config for type {datasource_type.full_type}. The plugin for this type is not a BuildDatasourcePlugin or CustomiseConfigProperties"
-        )
-
-
-def create_datasource_config_file(
-    project_dir: Path,
-    datasource_type: DatasourceType,
-    datasource_name: str,
-    config_content: dict[str, Any],
-    overwrite_existing: bool,
-) -> Path:
-    basic_config = {"type": datasource_type.subtype, "name": datasource_name}
-
-    return create_datasource_config_file_internal(
-        project_dir,
-        get_datasource_id_for_config_file(datasource_type, datasource_name),
-        to_yaml_string(basic_config | config_content),
-        overwrite_existing=overwrite_existing,
-    )
-
-
-def get_datasource_id_for_config_file(datasource_type: DatasourceType, datasource_name: str) -> DatasourceId:
-    return DatasourceId(
-        datasource_config_folder=datasource_type.config_folder,
-        datasource_name=datasource_name,
-        config_file_suffix=".yaml",
-    )

databao_context_engine/datasource_config/datasource_context.py

@@ -1,60 +0,0 @@
-import os
-from dataclasses import dataclass
-from pathlib import Path
-
-from databao_context_engine.project.layout import ProjectLayout
-from databao_context_engine.project.runs import get_run_dir, resolve_run_name
-from databao_context_engine.project.types import DatasourceId
-
-
-@dataclass(eq=True, frozen=True)
-class DatasourceContext:
-    datasource_id: DatasourceId
-    # TODO: Read the context as a BuildExecutionResult instead of a Yaml string?
-    context: str
-
-
-def get_datasource_context(
-    project_layout: ProjectLayout, datasource_id: DatasourceId, run_name: str | None = None
-) -> DatasourceContext:
-    run_dir = _resolve_run_dir(project_layout, run_name)
-
-    context_path = run_dir.joinpath(datasource_id.relative_path_to_context_file())
-    if not context_path.is_file():
-        raise ValueError(f"Context file not found for datasource {str(datasource_id)} in run {run_dir.name}")
-
-    context = context_path.read_text()
-    return DatasourceContext(datasource_id=datasource_id, context=context)
-
-
-def get_all_contexts(project_layout: ProjectLayout, run_name: str | None = None) -> list[DatasourceContext]:
-    run_dir = _resolve_run_dir(project_layout, run_name)
-
-    result = []
-    for main_type_dir in sorted((p for p in run_dir.iterdir() if p.is_dir()), key=lambda p: p.name.lower()):
-        for context_path in sorted(
-            (p for p in main_type_dir.iterdir() if p.suffix in [".yaml", ".yml"]), key=lambda p: p.name.lower()
-        ):
-            result.append(
-                DatasourceContext(
-                    # FIXME: The extension will always be yaml here even if the datasource is a file with a different extension
-                    datasource_id=DatasourceId.from_datasource_config_file_path(context_path),
-                    context=context_path.read_text(),
-                )
-            )
-
-    return result
-
-
-def get_context_header_for_datasource(datasource_id: DatasourceId) -> str:
-    return f"# ===== {str(datasource_id)} ====={os.linesep}"
-
-
-def _resolve_run_dir(project_layout: ProjectLayout, run_name: str | None) -> Path:
-    resolved_run_name = resolve_run_name(project_layout=project_layout, run_name=run_name)
-
-    run_dir = get_run_dir(project_dir=project_layout.project_dir, run_name=resolved_run_name)
-    if not run_dir.is_dir():
-        raise ValueError(f"Run {resolved_run_name} does not exist at {run_dir.resolve()}")
-
-    return run_dir

databao_context_engine/mcp/all_results_tool.py

@@ -1,5 +0,0 @@
-from databao_context_engine import DatabaoContextEngine
-
-
-def run_all_results_tool(databao_context_engine: DatabaoContextEngine, run_name: str | None) -> str:
-    return databao_context_engine.get_all_contexts_formatted(run_name=run_name)

databao_context_engine/mcp/retrieve_tool.py

@@ -1,22 +0,0 @@
-import datetime
-
-from databao_context_engine import DatabaoContextEngine
-
-
-def run_retrieve_tool(
-    *, databao_context_engine: DatabaoContextEngine, run_name: str | None, text: str, limit: int | None = None
-) -> str:
-    """
-    Execute the retrieve flow for MCP and return the matching display texts
-    Adds the current date to the end
-    """
-
-    retrieve_results = databao_context_engine.search_context(
-        retrieve_text=text, run_name=run_name, limit=limit, export_to_file=False
-    )
-
-    display_results = [context_search_result.context_result for context_search_result in retrieve_results]
-
-    display_results.append(f"\nToday's date is {datetime.date.today()}")
-
-    return "\n".join(display_results)

databao_context_engine/project/runs.py

@@ -1,39 +0,0 @@
-from pathlib import Path
-
-from databao_context_engine.project.layout import ProjectLayout, get_output_dir
-from databao_context_engine.storage.connection import open_duckdb_connection
-from databao_context_engine.storage.repositories.factories import create_run_repository
-from databao_context_engine.storage.repositories.run_repository import RunRepository
-from databao_context_engine.system.properties import get_db_path
-
-
-def resolve_run_name(*, project_layout: ProjectLayout, run_name: str | None) -> str:
-    project_id = str(project_layout.read_config_file().project_id)
-
-    with open_duckdb_connection(get_db_path()) as conn:
-        run_repository = create_run_repository(conn)
-
-        return resolve_run_name_from_repo(run_repository=run_repository, project_id=project_id, run_name=run_name)
-
-
-def resolve_run_name_from_repo(*, run_repository: RunRepository, project_id: str, run_name: str | None) -> str:
-    if run_name is None:
-        latest = run_repository.get_latest_run_for_project(project_id=project_id)
-        if latest is None:
-            raise LookupError(f"No runs found for project '{project_id}'. Run a build first.")
-        return latest.run_name
-    else:
-        run = run_repository.get_by_run_name(project_id=project_id, run_name=run_name)
-        if run is None:
-            raise LookupError(f"Run '{run_name}' not found for project '{project_id}'.")
-        return run.run_name
-
-
-def get_run_dir(project_dir: Path, run_name: str) -> Path:
-    run_dir = get_output_dir(project_dir).joinpath(run_name)
-    if not run_dir.is_dir():
-        raise ValueError(
-            f"The run with name {run_name} doesn't exist in the project. [project_dir: {project_dir.resolve()}]"
-        )
-
-    return run_dir

databao_context_engine/retrieve_embeddings/internal/retrieve_wiring.py

@@ -1,29 +0,0 @@
-from databao_context_engine.llm.factory import create_ollama_embedding_provider, create_ollama_service
-from databao_context_engine.project.layout import ProjectLayout
-from databao_context_engine.retrieve_embeddings.internal.retrieve_runner import retrieve
-from databao_context_engine.services.factories import create_retrieve_service
-from databao_context_engine.storage.connection import open_duckdb_connection
-from databao_context_engine.storage.repositories.vector_search_repository import VectorSearchResult
-from databao_context_engine.system.properties import get_db_path
-
-
-def retrieve_embeddings(
-    project_layout: ProjectLayout,
-    retrieve_text: str,
-    run_name: str | None,
-    limit: int | None,
-    export_to_file: bool,
-) -> list[VectorSearchResult]:
-    with open_duckdb_connection(get_db_path()) as conn:
-        ollama_service = create_ollama_service()
-        embedding_provider = create_ollama_embedding_provider(ollama_service)
-        retrieve_service = create_retrieve_service(conn, embedding_provider=embedding_provider)
-        return retrieve(
-            project_dir=project_layout.project_dir,
-            retrieve_service=retrieve_service,
-            project_id=str(project_layout.read_config_file().project_id),
-            text=retrieve_text,
-            run_name=run_name,
-            limit=limit,
-            export_to_file=export_to_file,
-        )

databao_context_engine/retrieve_embeddings/public/api.py

@@ -1,3 +0,0 @@
-from databao_context_engine.retrieve_embeddings.internal.retrieve_wiring import retrieve_embeddings
-
-__all__ = ["retrieve_embeddings"]

databao_context_engine/services/run_name_policy.py

@@ -1,8 +0,0 @@
-from datetime import datetime
-
-
-class RunNamePolicy:
-    _RUN_DIR_PREFIX = "run-"
-
-    def build(self, *, run_started_at: datetime):
-        return f"{RunNamePolicy._RUN_DIR_PREFIX}{run_started_at.isoformat(timespec='seconds')}"