databao-context-engine 0.1.1__py3-none-any.whl → 0.1.5__py3-none-any.whl

databao_context_engine/storage/repositories/factories.py

@@ -1,24 +1,13 @@
-from _duckdb import DuckDBPyConnection
+from duckdb import DuckDBPyConnection
 
-from databao_context_engine.services.run_name_policy import RunNamePolicy
 from databao_context_engine.storage.repositories.chunk_repository import ChunkRepository
-from databao_context_engine.storage.repositories.datasource_run_repository import DatasourceRunRepository
 from databao_context_engine.storage.repositories.embedding_model_registry_repository import (
     EmbeddingModelRegistryRepository,
 )
 from databao_context_engine.storage.repositories.embedding_repository import EmbeddingRepository
-from databao_context_engine.storage.repositories.run_repository import RunRepository
 from databao_context_engine.storage.repositories.vector_search_repository import VectorSearchRepository
 
 
-def create_run_repository(conn: DuckDBPyConnection) -> RunRepository:
-    return RunRepository(conn, run_name_policy=RunNamePolicy())
-
-
-def create_datasource_run_repository(conn: DuckDBPyConnection) -> DatasourceRunRepository:
-    return DatasourceRunRepository(conn)
-
-
 def create_chunk_repository(conn: DuckDBPyConnection) -> ChunkRepository:
     return ChunkRepository(conn)
 

databao_context_engine/storage/repositories/vector_search_repository.py

@@ -1,10 +1,11 @@
 from collections.abc import Sequence
 from dataclasses import dataclass
+from typing import Any
 
 import duckdb
 
+from databao_context_engine.datasources.types import DatasourceId
 from databao_context_engine.pluginlib.build_plugin import DatasourceType
-from databao_context_engine.project.types import DatasourceId
 
 
 @dataclass(kw_only=True, frozen=True)
@@ -23,32 +24,38 @@ class VectorSearchRepository:
         self._conn = conn
 
     def get_display_texts_by_similarity(
-        self, *, table_name: str, run_id: int, retrieve_vec: Sequence[float], dimension: int, limit: int
+        self,
+        *,
+        table_name: str,
+        retrieve_vec: Sequence[float],
+        dimension: int,
+        limit: int,
+        datasource_ids: list[DatasourceId] | None = None,
     ) -> list[VectorSearchResult]:
-        """
-        Read only similarity search on a specific embedding shard table.
-        Returns the display text for the closest matches in a given run
-        """
+        """Read only similarity search on a specific embedding shard table."""
+        params: list[Any] = [list(retrieve_vec), self._DEFAULT_DISTANCE_THRESHOLD, list(retrieve_vec), limit]
+        if datasource_ids:
+            params.append([str(datasource_id) for datasource_id in datasource_ids])
+
         rows = self._conn.execute(
             f"""
             SELECT
                 COALESCE(c.display_text, c.embeddable_text) AS display_text,
-                c.embeddable_text AS embeddable_text,
-                array_cosine_distance(e.vec, CAST(? AS FLOAT[{dimension}])) AS cosine_distance,
-                dr.full_type,
-                dr.source_id,
+                c.embeddable_text,
+                array_cosine_distance(e.vec, CAST($1 AS FLOAT[{dimension}])) AS cosine_distance,
+                c.full_type,
+                c.datasource_id,
             FROM
                 {table_name} e
                 JOIN chunk c ON e.chunk_id = c.chunk_id
-                JOIN datasource_run dr ON c.datasource_run_id = dr.datasource_run_id
             WHERE
-                dr.run_id = ?
-                AND cosine_distance < ?
+                cosine_distance < $2
+                {"AND c.datasource_id IN $5" if datasource_ids else ""}
             ORDER BY
-                array_cosine_distance(e.vec, CAST(? AS FLOAT[{dimension}])) ASC
-            LIMIT ?
+                array_cosine_distance(e.vec, CAST($3 AS FLOAT[{dimension}])) ASC
+            LIMIT $4
             """,
-            [list(retrieve_vec), run_id, self._DEFAULT_DISTANCE_THRESHOLD, list(retrieve_vec), limit],
+            params,
         ).fetchall()
 
         return [

databao_context_engine/system/properties.py

@@ -1,6 +1,8 @@
 import os
 from pathlib import Path
 
+from databao_context_engine.project.layout import get_output_dir
+
 # it's private, so it doesn't get imported directy. This value is mocked in tests
 _dce_path = Path(os.getenv("DATABAO_CONTEXT_ENGINE_PATH") or "~/.dce").expanduser().resolve()
 
@@ -9,5 +11,5 @@ def get_dce_path() -> Path:
     return _dce_path
 
 
-def get_db_path() -> Path:
-    return get_dce_path() / "dce.duckdb"
+def get_db_path(project_dir: Path) -> Path:
+    return get_output_dir(project_dir) / "dce.duckdb"

databao_context_engine-0.1.5.dist-info/METADATA

@@ -0,0 +1,228 @@
+Metadata-Version: 2.3
+Name: databao-context-engine
+Version: 0.1.5
+Summary: Semantic context for your LLMs — generated automatically
+Requires-Dist: click>=8.3.0
+Requires-Dist: duckdb>=1.4.3
+Requires-Dist: pyyaml>=6.0.3
+Requires-Dist: requests>=2.32.5
+Requires-Dist: mcp>=1.23.3
+Requires-Dist: pydantic>=2.12.4
+Requires-Dist: jinja2>=3.1.6
+Requires-Dist: pyathena>=3.25.0 ; extra == 'athena'
+Requires-Dist: clickhouse-connect>=0.10.0 ; extra == 'clickhouse'
+Requires-Dist: mssql-python>=1.0.0 ; extra == 'mssql'
+Requires-Dist: pymysql>=1.1.2 ; extra == 'mysql'
+Requires-Dist: asyncpg>=0.31.0 ; extra == 'postgresql'
+Requires-Dist: snowflake-connector-python>=4.2.0 ; extra == 'snowflake'
+Requires-Python: >=3.12
+Provides-Extra: athena
+Provides-Extra: clickhouse
+Provides-Extra: mssql
+Provides-Extra: mysql
+Provides-Extra: postgresql
+Provides-Extra: snowflake
+Description-Content-Type: text/markdown
+
+[![official project](https://jb.gg/badges/official.svg)](https://github.com/JetBrains#jetbrains-on-github)
+[![PyPI version](https://img.shields.io/pypi/v/databao-context-engine.svg)](https://pypi.org/project/databao-context-engine)
+[![License](https://img.shields.io/badge/license-Apache%202.0-blue.svg)](https://github.com/JetBrains/databao-context-engine?tab=License-1-ov-file)
+
+[//]: # ([![Python versions](https://img.shields.io/pypi/pyversions/databao-context-engine.svg)](https://pypi.org/project/databao-context-engine/))
+
+
+<h1 align="center">Databao Context Engine</h1>
+<p align="center">
+ <b>Semantic context for your LLMs — generated automatically.</b><br/>
+ No more copying schemas. No manual documentation. Just accurate answers.
+</p>
+<p align="center"> 
+  <a href="https://databao.app">Website</a> •
+  <a href="#quickstart">Quickstart</a> •
+  <a href="#supported-data-sources">Data Sources</a> •
+  <a href="#contributing">Contributing</a>
+</p>
+
+---
+
+## What is Databao Context Engine?
+
+Databao Context Engine is a CLI tool that **automatically generates governed semantic context** from your databases, BI tools, documents, and spreadsheets.
+
+Integrate it with any LLM to deliver **accurate, context-aware answers** — without copying schemas or writing documentation by hand.
+
+```
+Your data sources → Context Engine → Unified semantic graph → Any LLM
+```
+
+## Why choose Databao Context Engine?
+
+| Feature                    | What it means for you                                          |
+|----------------------------|----------------------------------------------------------------|
+| **Auto-generated context** | Extracts schemas, relationships, and semantics automatically   |
+| **Runs locally**           | Your data never leaves your environment                        |
+| **MCP integration**        | Works with Claude Desktop, Cursor, and any MCP-compatible tool |
+| **Multiple sources**       | Databases, dbt projects, spreadsheets, documents               |
+| **Built-in benchmarks**    | Measure and improve context quality over time                  |
+| **LLM agnostic**           | OpenAI, Anthropic, Ollama, Gemini — use any model              |
+| **Governed & versioned**   | Track, version, and share context across your team             |
+| **Dynamic or static**      | Serve context via MCP server or export as artifact             |
+
+## Installation
+
+Databao Context Engine is [available on PyPI](https://pypi.org/project/databao-context-engine/) and can be installed with uv, pip, or another package manage.
+
+### Using uv
+
+1. Install Databao Context Engine:
+
+   ```bash
+   uv tool install databao-context-engine
+   ```
+
+1. Add it to your PATH:
+
+   ```bash
+   uv tool update-shell
+   ```
+
+1. Verify the installation:
+
+   ```bash
+   dce --help
+   ```
+
+### Using pip
+
+1. Install Databao Context Engine:
+
+   ```bash
+   pip install databao-context-engine
+   ```
+
+1. Verify the installation:
+
+   ```bash
+   dce --help
+   ```
+
+##  Supported data sources
+
+* <img src="https://cdn.simpleicons.org/postgresql/316192" width="16" height="16" alt=""> PostgreSQL
+* <img src="https://cdn.simpleicons.org/mysql/4479A1" width="16" height="16" alt=""> MySQL
+* <img src="https://cdn.simpleicons.org/sqlite/003B57" width="16" height="16" alt=""> SQLite
+* <img src="https://cdn.simpleicons.org/duckdb/FFF000" width="16" height="16" alt=""> DuckDB
+* <img src="https://cdn.simpleicons.org/dbt/FF694B" width="16" height="16" alt=""> dbt projects
+* 📄 Documents & spreadsheets *(coming soon)*
+
+##  Supported LLMs
+
+| Provider      | Configuration                                |
+|---------------|----------------------------------------------|
+| **Ollama**    | `languageModel: OLLAMA`: runs locally, free  |
+| **OpenAI**    | `languageModel: OPENAI`: requires an API key |
+| **Anthropic** | `languageModel: CLAUDE`: requires an API key |
+| **Google**    | `languageModel: GEMINI`: requires an API key |
+
+## Quickstart
+
+### 1. Create a project
+
+1. Create a new directory for your project and navigate to it:
+
+   ```bash
+   mkdir dce-project && cd dce-project
+   ```
+
+1. Initialize a new project:
+
+   ```bash
+   dce init
+   ```
+
+### 2. Configure data sources
+
+1. When prompted, agree to create a new datasource.
+   You can also use the `dce datasource add` command.
+
+1. Provide the data source type and its name.
+
+1. Open the config file that was created for you in your editor and fill in the connection details.
+
+1. Repeat these steps for all data sources you want to include in your project.
+
+1. If you have data in Markdown or text files,
+   you can add them to the `dce/src/files` directory.
+
+### 3. Build context
+
+1. To build the context, run the following command:
+
+   ```bash
+   dce build
+   ```
+
+### 4. Use Context with Your LLM
+
+**Option A: Dynamic via MCP Server**
+
+Databao Context Engine exposes the context through a local MCP Server, so your agent can access the latest context at runtime.
+
+1. In **Claude Desktop**, **Cursor**, or another MCP-compatible agent, add the following configuration.
+   Replace `dce-project/` with the path to your project directory:
+  
+   ```json 
+   # claude_desktop_config.json, mcp.json, or similar
+   
+   {
+     "mcpServers": {
+       "dce": {
+         "command": "dce mcp",
+         "args": ["--project-dir", "dce-project/"]
+       }
+     }
+   }
+   ```
+
+1. Save the file and restart your agent.
+
+1. Open a new chat, in the chat window, select the `dce` server, and ask questions related to your project context.
+
+**Option B: Static artifact**
+
+Even if you don’t have Claude or Cursor installed on your local machine,
+you can still use the context built by Databao Context Engine by pasting it directly into your chat with an AI assistant.
+
+1. Navigate to `dce-project/output/` and open the directory with the latest run.
+
+1. Attach the `all_results.yaml` file to your chat with the AI assistant or copy and paste its contents into your chat.
+
+##  Contributing
+
+We’d love your help! Here’s how to get involved:
+
+- ⭐ **Star this repo** — it helps others find us!
+- 🐛 **Found a bug?** [Open an issue](https://github.com/JetBrains/databao-context-engine/issues)
+- 💡 **Have an idea?** We’re all ears — create a feature request
+- 👍 **Upvote issues** you care about — helps us prioritize
+- 🔧 **Submit a PR**
+- 📝 **Improve docs** — typos, examples, tutorials — everything helps!
+
+New to open source? No worries! We're friendly and happy to help you get started. 🌱
+
+For more details, see [CONTRIBUTING](CONTRIBUTING.md).
+
+## 📄 License
+
+Apache 2.0 — use it however you want. See the [LICENSE](LICENSE.md) file for details.
+
+---
+
+<p align="center">
+ <b>Like Databao Context Engine?</b> Give us a ⭐ — it means a lot!
+</p>
+
+<p align="center">
+ <a href="https://databao.app">Website</a> •
+ <a href="https://discord.gg/databao">Discord</a>
+</p>

databao_context_engine-0.1.5.dist-info/RECORD

@@ -0,0 +1,135 @@
+databao_context_engine/__init__.py,sha256=sPhGoLZhsFvf1n-WLt9gFUL1IZegCQJPkOZrcrJYn2c,2239
+databao_context_engine/build_sources/__init__.py,sha256=FteGp3MvSuX_fh-Fx1pq-jOjFcjYKQSXFSll4tQffvk,224
+databao_context_engine/build_sources/build_runner.py,sha256=aSxp99R4tbnbHlEDFrg8iw4Bi1dGvO_5uTO-mIrB080,3986
+databao_context_engine/build_sources/build_service.py,sha256=eywJzpvZUVrj-JYxXii_kxhTBzVUT0xfbNtdAFcpGWQ,1624
+databao_context_engine/build_sources/build_wiring.py,sha256=Hfdk0CF0LuGiGGQjOl7wVpVBnYRgrLvT_iLUq-Xljng,3174
+databao_context_engine/build_sources/export_results.py,sha256=E3HP4uaQKYDrmAJKFWwDT190j7ZKfO791_pjKebTMQM,1773
+databao_context_engine/build_sources/plugin_execution.py,sha256=wfMCF_9ixuEnV2VMhCDwOwBN5bi8Yz6ap5ayau2TAYk,2090
+databao_context_engine/cli/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+databao_context_engine/cli/add_datasource_config.py,sha256=fK1MIaTCoz1jurldJZD_BpPIgc6kr-NaAyVjVgjsOTU,5523
+databao_context_engine/cli/commands.py,sha256=VJOHB3H28wNjOUuFC8hhF8-rnlIFjsZMlNT3C2ZbIUY,7807
+databao_context_engine/cli/datasources.py,sha256=kHbXhEnDVaYXTRAE3WeaDGohH1Y9cwyYjjcycQ8eh1U,2376
+databao_context_engine/cli/info.py,sha256=2iegfvRAxMae3KkWMDu-BOnljeJMVQ5u8P5k_8nOKpc,1203
+databao_context_engine/config/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+databao_context_engine/config/log_config.yaml,sha256=hH2NPumufcyig8L-7Z_DbLQ88_G2N6CAEl5fvHafeAw,360
+databao_context_engine/config/logging.py,sha256=66k7x3ftEdHx-CKqbOss4pqZfGn4nMFv7AzVPs9aVnY,1525
+databao_context_engine/databao_context_engine.py,sha256=l8F1JOzDDZYvIannM6Yq_tGR--G23XzjTVZn6dnjSwY,4966
+databao_context_engine/databao_context_project_manager.py,sha256=xF_ZQUd8NxnP6ow-y6YCGxzPO9Gfy5TTSqIeC5N9XOU,8288
+databao_context_engine/datasources/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+databao_context_engine/datasources/check_config.py,sha256=YsrWFCwdJv3jvXG_tTuCRr129351GmRTKB0BRYXxtbo,5407
+databao_context_engine/datasources/datasource_context.py,sha256=uRuviRJUsZdHcNan5DYc3ZBpw2SSsHIMuZjR4PbXUkg,3687
+databao_context_engine/datasources/datasource_discovery.py,sha256=ZrjVyKIk93XbBLTVnnNAR4CFBFflmPZgvmBfOq83hZs,5573
+databao_context_engine/datasources/types.py,sha256=gF2BfNC-NSSvXEg-mefGPBKCGsLhUiDySIv56SsvbIM,7449
+databao_context_engine/event_journal/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+databao_context_engine/event_journal/writer.py,sha256=abkl2SPHYzO8XAUsjbtrcXm6WQk15zWpXhYm86qb6G0,986
+databao_context_engine/generate_configs_schemas.py,sha256=rUjSg_SnF7C8-OJ8dNwPpbEdPQqxnF2X8qz3LwIo0e0,3007
+databao_context_engine/init_project.py,sha256=enLvYOd5MpGYVdWly7DuLMzPLUEE6130bEuxh0YZft8,1496
+databao_context_engine/introspection/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+databao_context_engine/introspection/property_extract.py,sha256=DlTdqvysEmYMsmeJMdaxQuzbBalMxTq7XLqMSwecYCg,7891
+databao_context_engine/llm/__init__.py,sha256=FAGvpG3kBp4ssNYre8ytTtPeCF6XdZYNP7qpEUhpNAk,366
+databao_context_engine/llm/api.py,sha256=eizZD1PGwkJbj0CFHDtdqcPWp3OdbbUSWIIcF3M3MSo,2091
+databao_context_engine/llm/config.py,sha256=geIygR-9maeKbvmcO5J3iNZXKS4LZc7zpKYyaH8vCoI,463
+databao_context_engine/llm/descriptions/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+databao_context_engine/llm/descriptions/ollama.py,sha256=MkRajlkIsVvLzQvkVBsGwjHqj5lNylexu5o-HQSnbQQ,622
+databao_context_engine/llm/descriptions/provider.py,sha256=nojEI0l55TfFuNooQwGNvfa2k6A1vZ1Oi3PBLAr9eFo,227
+databao_context_engine/llm/embeddings/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+databao_context_engine/llm/embeddings/ollama.py,sha256=GQzlMBRJGJe_xreY9-Y3REBFWWaOyKhiGQxWxrR6xfY,949
+databao_context_engine/llm/embeddings/provider.py,sha256=IVrOptwad2Fr1wf1SBOM3-7y66k08mOZnga1NA6yhbA,300
+databao_context_engine/llm/errors.py,sha256=Y4CVHYJKkn2oHnFWVlVL1QJl4gtvW4z11V3FB9qp3Cc,402
+databao_context_engine/llm/factory.py,sha256=Ct6IyC984POktLNEfOyrIw9y1IqZ6YNg4zSx16L7VhQ,1888
+databao_context_engine/llm/install.py,sha256=1NqpoERmLvX3maFFZQHZ0I7uW-Moi3oyySQoTavZqbM,7082
+databao_context_engine/llm/runtime.py,sha256=uEIiGb9OkjGwlZH6arb3lmD3NHRByh1nngJW-4EWQn0,2343
+databao_context_engine/llm/service.py,sha256=rM_DrF2F7NylxBhtfVGporUcv91eR_nF1mF--Zxh2tc,5978
+databao_context_engine/main.py,sha256=3gWwEc9GZJUFANJtW9Fx4smVzib_sDtFkBGPILjisaI,350
+databao_context_engine/mcp/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+databao_context_engine/mcp/mcp_runner.py,sha256=bnB2a-_P1zOlDBqpyFIq4n2hWkfwqb6G7dHb3-3Q2I0,468
+databao_context_engine/mcp/mcp_server.py,sha256=Z2pesefHCFhzfSdPa8OsruxynE28dmEKuEmwx3N18kM,2179
+databao_context_engine/plugin_loader.py,sha256=KIHUwxMcWdyPvZlHFxIDGwIMILTm6mO_gJAzw1UIA00,4784
+databao_context_engine/pluginlib/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+databao_context_engine/pluginlib/build_plugin.py,sha256=y-guT2X7wS8Yn1C6iBWLF_HDA4NYVFJsQ-oe6IgVvDo,2831
+databao_context_engine/pluginlib/config.py,sha256=fAu1AuxSFI6LOoAyvwuFE5p11bOsHp9EZzn2ST8DrUI,1534
+databao_context_engine/pluginlib/plugin_utils.py,sha256=f76ksdMUG3Wy4KOytUYxLMrq9iOHytu3QF8wc5NzzJE,2471
+databao_context_engine/plugins/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+databao_context_engine/plugins/databases/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+databao_context_engine/plugins/databases/athena/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+databao_context_engine/plugins/databases/athena/athena_db_plugin.py,sha256=1Jn3kqPExKUxvj4oXqgJQwRoLiu_3uKgSzEESvGm_Qs,463
+databao_context_engine/plugins/databases/athena/athena_introspector.py,sha256=PCdU8pHEAr6UliLwDgLNEqiN9MqwJ0k94H2f6TgBBmA,5627
+databao_context_engine/plugins/databases/base_db_plugin.py,sha256=ivCgxqb5wBjCgGuPxV5-oFF8R0h-zcAt1ckgTQ87V4Q,1680
+databao_context_engine/plugins/databases/base_introspector.py,sha256=08L6nmIBJT3rfWB2fGdm7jSrNrW4un2MMMLt5shorKo,5752
+databao_context_engine/plugins/databases/clickhouse/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+databao_context_engine/plugins/databases/clickhouse/clickhouse_db_plugin.py,sha256=K3cWnZFyKGpFfg5VBJ0016C9H9DuADIBJdomzH9T49E,520
+databao_context_engine/plugins/databases/clickhouse/clickhouse_introspector.py,sha256=wQbriPsb1HHbx7b7P86K-dPDGIziXxF7rjM_gUy_cUs,6267
+databao_context_engine/plugins/databases/database_chunker.py,sha256=dn9Mb9Sw4f8IwBe9RtukQGT0yyKLQt5i4tjvsrFh1ew,2100
+databao_context_engine/plugins/databases/databases_types.py,sha256=kYZEaFa_cnROg_n1k7tIonoC2VunLBe5SQR47GOt_8k,2579
+databao_context_engine/plugins/databases/duckdb/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+databao_context_engine/plugins/databases/duckdb/duckdb_db_plugin.py,sha256=UyFp-MihNj562Mx9KcogFX33yNbNUMX_fdmvF6MrJ1M,458
+databao_context_engine/plugins/databases/duckdb/duckdb_introspector.py,sha256=dhZBfKKpyETOrfbQBIM8urfztH_qfhTrvzNFlIesC6c,11260
+databao_context_engine/plugins/databases/introspection_model_builder.py,sha256=-i7e4jqwdcsu3NnS-LHVSH_DrG3riN09KX3SDHAGF0c,10350
+databao_context_engine/plugins/databases/introspection_scope.py,sha256=0BZAAmVTiHifOzeuQkCsGKrGRnZCPHRCLqam5nQqr88,2225
+databao_context_engine/plugins/databases/introspection_scope_matcher.py,sha256=gseF4j-SD-7SSPMDlDqHUOTI3tPSaOGKJgp42WDVlsY,3605
+databao_context_engine/plugins/databases/mssql/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+databao_context_engine/plugins/databases/mssql/mssql_db_plugin.py,sha256=KV_KQw2f_wIj0B7_rtW0XGcesSOCqPy5A9FTYEDY7PI,452
+databao_context_engine/plugins/databases/mssql/mssql_introspector.py,sha256=bqaERs8uRZlTfVQ91A9FygHpDX-KmGZXNUp8iDgxPdg,18704
+databao_context_engine/plugins/databases/mysql/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+databao_context_engine/plugins/databases/mysql/mysql_db_plugin.py,sha256=40c6wznmQjU7bSs1kStS4WgNyoZ44wU1Ik9uLdvDqfI,452
+databao_context_engine/plugins/databases/mysql/mysql_introspector.py,sha256=-OlXluCVTUTTZzRJPAVvcHeRdocopHcDKgBFWXF9kGo,14461
+databao_context_engine/plugins/databases/postgresql/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+databao_context_engine/plugins/databases/postgresql/postgresql_db_plugin.py,sha256=VHCtLkoZhgbw631Z-cjcZOyI7dQZii-1fkdCJmntLWE,510
+databao_context_engine/plugins/databases/postgresql/postgresql_introspector.py,sha256=OoaVLgpLOB4HDTU21S5upNyZD3H3B-z5fJq5NMOtakU,17275
+databao_context_engine/plugins/databases/snowflake/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+databao_context_engine/plugins/databases/snowflake/snowflake_db_plugin.py,sha256=DCIpicjpS9qkd2EfumO7UDG6RQERW3s3FDPDSue7ehg,509
+databao_context_engine/plugins/databases/snowflake/snowflake_introspector.py,sha256=8-a478TSu3uL_1z5HeS9nnF4VxZ9DZNnjH1coyqINM0,11962
+databao_context_engine/plugins/databases/sqlite/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+databao_context_engine/plugins/databases/sqlite/sqlite_db_plugin.py,sha256=MI896G7Yq8NcQ8sTWErAFD7YK8qfDpZZprxoszJ46l8,460
+databao_context_engine/plugins/databases/sqlite/sqlite_introspector.py,sha256=kKFNzclp1NmQ6BF3ylGn86R5PuQh6aqevj6E7_zAKmQ,8361
+databao_context_engine/plugins/duckdb_tools.py,sha256=46rctnTxDPAhHtaiTp1DxMuuDuRKrtKWJFSSM2w7uUU,645
+databao_context_engine/plugins/files/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+databao_context_engine/plugins/files/unstructured_files_plugin.py,sha256=eqs1anQhYBZh7xu4CwhfkqXQjGE5gJnKEwyJbtUR78E,2384
+databao_context_engine/plugins/plugin_loader.py,sha256=8j7JAKqtG6_VjX3GbfAD6kjrWuuJ7NnC2nnHPEujux8,4074
+databao_context_engine/plugins/resources/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+databao_context_engine/plugins/resources/parquet_chunker.py,sha256=R9WCOBqpKRTVN6t5eeOm_mmnKBOxvjIiQ9zTc8vnUb4,848
+databao_context_engine/plugins/resources/parquet_introspector.py,sha256=Cn_yh6E-dOTOZstlavEGAsV6ZRKZXJraVAl_pzJJuGs,5629
+databao_context_engine/plugins/resources/parquet_plugin.py,sha256=FVRdAqhZkEBvUsW1CkYH4yPPs3P1pRsofR8PdHN26AE,1128
+databao_context_engine/project/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+databao_context_engine/project/info.py,sha256=3chVSZ4pV2EsYJSeVfOYTK746POlJImciIBOvrQuYbc,2479
+databao_context_engine/project/init_project.py,sha256=QiLalH--BMQGB-c2WBbPAUKBinDbO0cJkmilUfqPAuI,3970
+databao_context_engine/project/layout.py,sha256=3ttCJDLMh4AS1tBRcSROgT7hG6uOqkI1QGsQEcfGsqg,4087
+databao_context_engine/project/project_config.py,sha256=rI-Wkll7lca7RlYIaFScs5CIKIZ8uujtKRxSjibGIt8,1132
+databao_context_engine/project/resources/examples/src/databases/example_postgres.yaml,sha256=5o6jF_zR4WSvS9QcZnGW-S2FmahsjNp-tMbAkzbNabo,139
+databao_context_engine/project/resources/examples/src/files/documentation.md,sha256=k5G5lrN4S_ie2o7CBxXgLiWGIGtb3v-77c_3n4C7Lvw,1090
+databao_context_engine/project/resources/examples/src/files/notes.txt,sha256=0T1u8lIwIRwNopEsfWmqfIbhfRl-DN3XuiIN7suRAiE,3668
+databao_context_engine/retrieve_embeddings/__init__.py,sha256=__aOhCc3sBfLtjxz0hlVGuSYUMRsEM3PucPYNC6LVq0,126
+databao_context_engine/retrieve_embeddings/retrieve_runner.py,sha256=s7XwPkMupBgN9JADFaHsytM5ztujyqujB7qdmXKv7ms,585
+databao_context_engine/retrieve_embeddings/retrieve_service.py,sha256=0KOSaxyj6DfzLbHtcvxFPUjXO4MD5t91RP95rsVuE6Q,2275
+databao_context_engine/retrieve_embeddings/retrieve_wiring.py,sha256=GQ9Eh6XyQ9MdG8Kb0H72XPj_8-gPDLSM_Bs0NdmSMX4,2012
+databao_context_engine/serialization/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+databao_context_engine/serialization/yaml.py,sha256=ke-0iHGaA2eynuGf-tG9IZip56XFfMDrNswzi5rNaDE,1219
+databao_context_engine/services/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+databao_context_engine/services/chunk_embedding_service.py,sha256=Dy25u2QSSdq03v1zs2bjjm8Sze-VEd12F49gzoKXjOA,4924
+databao_context_engine/services/embedding_shard_resolver.py,sha256=6oOVm6QJb74lEIbYXijWMqISTx1VfmKQaQjdSHr8zDA,2289
+databao_context_engine/services/factories.py,sha256=IoRmJk9a7q2eszwz_zfqE2hJDOwpB76SmmmgR0DI-qY,1830
+databao_context_engine/services/models.py,sha256=wHmk4eheoSopXkCs3v2ggoDAGoTPdHYcADMZ4gdtSCI,299
+databao_context_engine/services/persistence_service.py,sha256=Kcu4rw-TCqyHiGW1h5rYZfe3F-HZTn0tRAM4_czzEAc,2252
+databao_context_engine/services/table_name_policy.py,sha256=q0scAQ_ZoTrV8V0J6cmxvsMYxTcmUlAqEWpzZUitpLQ,567
+databao_context_engine/storage/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+databao_context_engine/storage/connection.py,sha256=UGnJOBJ83jDAmN3fmjVZI60qUw5cAC3YNLToJZLOhGQ,941
+databao_context_engine/storage/exceptions/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+databao_context_engine/storage/exceptions/exceptions.py,sha256=zwJ8n-ekMhIH-bs9dvYqQuP4VfQ9rt0jdsptoojA5oU,174
+databao_context_engine/storage/migrate.py,sha256=rbZDhQEzykQyfGflJodGv-1ohs43oGSGYXEyT6jTVeo,4617
+databao_context_engine/storage/migrations/V01__init.sql,sha256=hJvUoPIW-rQQB5DSDrPjGJQoey9bpDRz0FFoUyTe9Qo,1368
+databao_context_engine/storage/models.py,sha256=TmoeyOhrFL9mu-C43XUtqOWGhkQu2h-sZQdYCjui5HM,580
+databao_context_engine/storage/repositories/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+databao_context_engine/storage/repositories/chunk_repository.py,sha256=nwDF8NeCwQJLlwtlvqILUccmWmQTfpg6Uf_WCDXm564,3681
+databao_context_engine/storage/repositories/embedding_model_registry_repository.py,sha256=g91WCflYRc5jPDbW9RBp5HxWFsSIta4n8OYNwgoLv9c,2413
+databao_context_engine/storage/repositories/embedding_repository.py,sha256=UCvr5mf2jTJgX_uMdLIMiN5NWa8Zn1U3HEr19TQRXAw,3278
+databao_context_engine/storage/repositories/factories.py,sha256=YUkVrHyf5UV2rZ7H97Fi_n0ZYxTwXy8_w5zJnMkO8Lw,970
+databao_context_engine/storage/repositories/vector_search_repository.py,sha256=jqCRZ18FKuJFVMsZtHRs19RejVlbZ4VDZY9E6ryeQfM,2289
+databao_context_engine/storage/transaction.py,sha256=QpfE1VtgDd3_7OBzFYI3HPJJ7ESXkgCdcWukFkAjbSg,270
+databao_context_engine/system/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+databao_context_engine/system/properties.py,sha256=mQ7-_PZeYSESYn1cMUQ0IK7rJEnbhc7t4WesFjAgo-Q,429
+databao_context_engine/templating/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+databao_context_engine/templating/renderer.py,sha256=W2-0IGStAp6oxANmsKs_Z-UoIR6Gt_c4ILYFa3Hruo4,662
+databao_context_engine-0.1.5.dist-info/WHEEL,sha256=5w2T7AS2mz1-rW9CNagNYWRCaB0iQqBMYLwKdlgiR4Q,78
+databao_context_engine-0.1.5.dist-info/entry_points.txt,sha256=5EeQJ1W8zEFh4HuF1bs2zBeoP408oiwuM9UrkJiurgI,138
+databao_context_engine-0.1.5.dist-info/METADATA,sha256=HfJBADmvbXEEK710S6zzybfWl-mqnDXfm_18LIgh7kk,7773
+databao_context_engine-0.1.5.dist-info/RECORD,,

{databao_context_engine-0.1.1.dist-info → databao_context_engine-0.1.5.dist-info}/WHEEL

@@ -1,4 +1,4 @@
 Wheel-Version: 1.0
-Generator: uv 0.9.24
+Generator: uv 0.9.7
 Root-Is-Purelib: true
 Tag: py3-none-any

databao_context_engine/build_sources/internal/build_service.py

@@ -1,77 +0,0 @@
-from __future__ import annotations
-
-import logging
-from datetime import datetime
-
-from databao_context_engine.build_sources.internal.plugin_execution import BuiltDatasourceContext, execute
-from databao_context_engine.pluginlib.build_plugin import (
-    BuildPlugin,
-)
-from databao_context_engine.project.types import PreparedDatasource
-from databao_context_engine.serialisation.yaml import to_yaml_string
-from databao_context_engine.services.chunk_embedding_service import ChunkEmbeddingService
-from databao_context_engine.storage.models import RunDTO
-from databao_context_engine.storage.repositories.datasource_run_repository import DatasourceRunRepository
-from databao_context_engine.storage.repositories.run_repository import RunRepository
-
-logger = logging.getLogger(__name__)
-
-
-class BuildService:
-    def __init__(
-        self,
-        *,
-        run_repo: RunRepository,
-        datasource_run_repo: DatasourceRunRepository,
-        chunk_embedding_service: ChunkEmbeddingService,
-    ) -> None:
-        self._run_repo = run_repo
-        self._datasource_run_repo = datasource_run_repo
-        self._chunk_embedding_service = chunk_embedding_service
-
-    def start_run(self, *, project_id: str, dce_version: str) -> RunDTO:
-        """
-        Create a new run row and return (run_id, started_at).
-        """
-        return self._run_repo.create(project_id=project_id, dce_version=dce_version)
-
-    def finalize_run(self, *, run_id: int):
-        """
-        Mark the run as complete (sets ended_at).
-        """
-        self._run_repo.update(run_id=run_id, ended_at=datetime.now())
-
-    def process_prepared_source(
-        self,
-        *,
-        run_id: int,
-        prepared_source: PreparedDatasource,
-        plugin: BuildPlugin,
-    ) -> BuiltDatasourceContext:
-        """
-        Process a single source.
-
-        1) Execute the plugin
-        2) Divide the results into chunks
-        3) Embed and persist the chunks
-        """
-        result = execute(prepared_source, plugin)
-
-        chunks = plugin.divide_context_into_chunks(result.context)
-        if not chunks:
-            logger.info("No chunks for %s — skipping.", prepared_source.path.name)
-            return result
-
-        datasource_run = self._datasource_run_repo.create(
-            run_id=run_id,
-            plugin=plugin.name,
-            full_type=prepared_source.datasource_type.full_type,
-            source_id=result.datasource_id,
-            storage_directory=str(prepared_source.path.parent),
-        )
-
-        self._chunk_embedding_service.embed_chunks(
-            datasource_run_id=datasource_run.datasource_run_id, chunks=chunks, result=to_yaml_string(result.context)
-        )
-
-        return result

databao_context_engine/build_sources/internal/build_wiring.py

@@ -1,52 +0,0 @@
-import logging
-from pathlib import Path
-
-from databao_context_engine.build_sources.internal.build_runner import BuildContextResult, build
-from databao_context_engine.llm.factory import (
-    create_ollama_description_provider,
-    create_ollama_embedding_provider,
-    create_ollama_service,
-)
-from databao_context_engine.project.info import get_dce_version
-from databao_context_engine.project.layout import ensure_project_dir
-from databao_context_engine.services.chunk_embedding_service import ChunkEmbeddingMode
-from databao_context_engine.services.factories import (
-    create_build_service,
-)
-from databao_context_engine.storage.connection import open_duckdb_connection
-from databao_context_engine.system.properties import get_db_path
-
-logger = logging.getLogger(__name__)
-
-
-def build_all_datasources(project_dir: Path, chunk_embedding_mode: ChunkEmbeddingMode) -> list[BuildContextResult]:
-    """
-    Public build entrypoint
-    - Instantiates the build service
-    - Delegates the actual build logic to the build runner
-    """
-    project_layout = ensure_project_dir(project_dir)
-
-    logger.debug(f"Starting to build datasources in project {project_dir.resolve()}")
-
-    with open_duckdb_connection(get_db_path()) as conn:
-        ollama_service = create_ollama_service()
-        embedding_provider = create_ollama_embedding_provider(ollama_service)
-        description_provider = (
-            create_ollama_description_provider(ollama_service)
-            if chunk_embedding_mode.should_generate_description()
-            else None
-        )
-        build_service = create_build_service(
-            conn,
-            embedding_provider=embedding_provider,
-            description_provider=description_provider,
-            chunk_embedding_mode=chunk_embedding_mode,
-        )
-        dce_config = project_layout.read_config_file()
-        return build(
-            project_dir=project_dir,
-            build_service=build_service,
-            project_id=str(dce_config.project_id),
-            dce_version=get_dce_version(),
-        )

databao_context_engine/build_sources/internal/export_results.py

@@ -1,43 +0,0 @@
-import logging
-from pathlib import Path
-
-from databao_context_engine.build_sources.internal.plugin_execution import BuiltDatasourceContext
-from databao_context_engine.datasource_config.datasource_context import get_context_header_for_datasource
-from databao_context_engine.project.layout import ALL_RESULTS_FILE_NAME, get_output_dir
-from databao_context_engine.project.types import DatasourceId
-from databao_context_engine.serialisation.yaml import write_yaml_to_stream
-
-logger = logging.getLogger(__name__)
-
-
-def create_run_dir(project_dir: Path, run_name: str) -> Path:
-    output_dir = get_output_dir(project_dir)
-
-    run_dir = output_dir.joinpath(run_name)
-    run_dir.mkdir(parents=True, exist_ok=False)
-
-    return run_dir
-
-
-def export_build_result(run_dir: Path, result: BuiltDatasourceContext) -> Path:
-    datasource_id = DatasourceId.from_string_repr(result.datasource_id)
-    export_file_path = run_dir.joinpath(datasource_id.relative_path_to_context_file())
-
-    # Make sure the parent folder exists
-    export_file_path.parent.mkdir(exist_ok=True)
-
-    with export_file_path.open("w") as export_file:
-        write_yaml_to_stream(data=result, file_stream=export_file)
-
-    logger.info(f"Exported result to {export_file_path.resolve()}")
-
-    return export_file_path
-
-
-def append_result_to_all_results(run_dir: Path, result: BuiltDatasourceContext):
-    path = run_dir.joinpath(ALL_RESULTS_FILE_NAME)
-    path.parent.mkdir(parents=True, exist_ok=True)
-    with path.open("a", encoding="utf-8") as export_file:
-        export_file.write(get_context_header_for_datasource(DatasourceId.from_string_repr(result.datasource_id)))
-        write_yaml_to_stream(data=result, file_stream=export_file)
-        export_file.write("\n")