google-cloud-db-context-engineering 0.5.1__tar.gz → 0.6.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {google_cloud_db_context_engineering-0.5.1 → google_cloud_db_context_engineering-0.6.0}/PKG-INFO +12 -14
- {google_cloud_db_context_engineering-0.5.1 → google_cloud_db_context_engineering-0.6.0}/README.md +11 -13
- {google_cloud_db_context_engineering-0.5.1 → google_cloud_db_context_engineering-0.6.0}/pyproject.toml +3 -3
- google_cloud_db_context_engineering-0.6.0/src/google/cloud/db_context_enrichment/main.py +197 -0
- {google_cloud_db_context_engineering-0.5.1 → google_cloud_db_context_engineering-0.6.0}/src/google_cloud_db_context_engineering.egg-info/PKG-INFO +12 -14
- {google_cloud_db_context_engineering-0.5.1 → google_cloud_db_context_engineering-0.6.0}/src/google_cloud_db_context_engineering.egg-info/SOURCES.txt +0 -14
- google_cloud_db_context_engineering-0.5.1/src/google/cloud/db_context_enrichment/bootstrap/__init__.py +0 -1
- google_cloud_db_context_engineering-0.5.1/src/google/cloud/db_context_enrichment/bootstrap/bootstrap_generator.py +0 -74
- google_cloud_db_context_engineering-0.5.1/src/google/cloud/db_context_enrichment/common/parameterizer.py +0 -192
- google_cloud_db_context_engineering-0.5.1/src/google/cloud/db_context_enrichment/facet/facet_generator.py +0 -68
- google_cloud_db_context_engineering-0.5.1/src/google/cloud/db_context_enrichment/main.py +0 -450
- google_cloud_db_context_engineering-0.5.1/src/google/cloud/db_context_enrichment/model/__init__.py +0 -0
- google_cloud_db_context_engineering-0.5.1/src/google/cloud/db_context_enrichment/prompts/__init__.py +0 -9
- google_cloud_db_context_engineering-0.5.1/src/google/cloud/db_context_enrichment/prompts/targeted_facets.py +0 -56
- google_cloud_db_context_engineering-0.5.1/src/google/cloud/db_context_enrichment/prompts/targeted_templates.py +0 -57
- google_cloud_db_context_engineering-0.5.1/src/google/cloud/db_context_enrichment/prompts/targeted_value_search.py +0 -93
- google_cloud_db_context_engineering-0.5.1/src/google/cloud/db_context_enrichment/template/__init__.py +0 -0
- google_cloud_db_context_engineering-0.5.1/src/google/cloud/db_context_enrichment/template/template_generator.py +0 -67
- google_cloud_db_context_engineering-0.5.1/src/google/cloud/db_context_enrichment/value_search/__init__.py +0 -0
- google_cloud_db_context_engineering-0.5.1/src/google/cloud/db_context_enrichment/value_search/generator.py +0 -91
- google_cloud_db_context_engineering-0.5.1/src/google/cloud/db_context_enrichment/value_search/match_templates.py +0 -267
- {google_cloud_db_context_engineering-0.5.1 → google_cloud_db_context_engineering-0.6.0}/LICENSE +0 -0
- {google_cloud_db_context_engineering-0.5.1 → google_cloud_db_context_engineering-0.6.0}/setup.cfg +0 -0
- {google_cloud_db_context_engineering-0.5.1 → google_cloud_db_context_engineering-0.6.0}/src/google/cloud/db_context_enrichment/__init__.py +0 -0
- {google_cloud_db_context_engineering-0.5.1 → google_cloud_db_context_engineering-0.6.0}/src/google/cloud/db_context_enrichment/common/__init__.py +0 -0
- {google_cloud_db_context_engineering-0.5.1 → google_cloud_db_context_engineering-0.6.0}/src/google/cloud/db_context_enrichment/common/config.py +0 -0
- {google_cloud_db_context_engineering-0.5.1 → google_cloud_db_context_engineering-0.6.0}/src/google/cloud/db_context_enrichment/common/context_mutator.py +0 -0
- {google_cloud_db_context_engineering-0.5.1 → google_cloud_db_context_engineering-0.6.0}/src/google/cloud/db_context_enrichment/dataset/__init__.py +0 -0
- {google_cloud_db_context_engineering-0.5.1 → google_cloud_db_context_engineering-0.6.0}/src/google/cloud/db_context_enrichment/dataset/dataset_generator.py +0 -0
- {google_cloud_db_context_engineering-0.5.1 → google_cloud_db_context_engineering-0.6.0}/src/google/cloud/db_context_enrichment/evaluate/__init__.py +0 -0
- {google_cloud_db_context_engineering-0.5.1 → google_cloud_db_context_engineering-0.6.0}/src/google/cloud/db_context_enrichment/evaluate/db_generators/__init__.py +0 -0
- {google_cloud_db_context_engineering-0.5.1 → google_cloud_db_context_engineering-0.6.0}/src/google/cloud/db_context_enrichment/evaluate/db_generators/alloydb.py +0 -0
- {google_cloud_db_context_engineering-0.5.1 → google_cloud_db_context_engineering-0.6.0}/src/google/cloud/db_context_enrichment/evaluate/db_generators/base.py +0 -0
- {google_cloud_db_context_engineering-0.5.1 → google_cloud_db_context_engineering-0.6.0}/src/google/cloud/db_context_enrichment/evaluate/db_generators/mysql.py +0 -0
- {google_cloud_db_context_engineering-0.5.1 → google_cloud_db_context_engineering-0.6.0}/src/google/cloud/db_context_enrichment/evaluate/db_generators/postgres.py +0 -0
- {google_cloud_db_context_engineering-0.5.1 → google_cloud_db_context_engineering-0.6.0}/src/google/cloud/db_context_enrichment/evaluate/db_generators/spanner.py +0 -0
- {google_cloud_db_context_engineering-0.5.1 → google_cloud_db_context_engineering-0.6.0}/src/google/cloud/db_context_enrichment/evaluate/evaluate_generator.py +0 -0
- {google_cloud_db_context_engineering-0.5.1 → google_cloud_db_context_engineering-0.6.0}/src/google/cloud/db_context_enrichment/evaluate/result_reader.py +0 -0
- {google_cloud_db_context_engineering-0.5.1/src/google/cloud/db_context_enrichment/facet → google_cloud_db_context_engineering-0.6.0/src/google/cloud/db_context_enrichment/model}/__init__.py +0 -0
- {google_cloud_db_context_engineering-0.5.1 → google_cloud_db_context_engineering-0.6.0}/src/google/cloud/db_context_enrichment/model/context.py +0 -0
- {google_cloud_db_context_engineering-0.5.1 → google_cloud_db_context_engineering-0.6.0}/src/google_cloud_db_context_engineering.egg-info/dependency_links.txt +0 -0
- {google_cloud_db_context_engineering-0.5.1 → google_cloud_db_context_engineering-0.6.0}/src/google_cloud_db_context_engineering.egg-info/entry_points.txt +0 -0
- {google_cloud_db_context_engineering-0.5.1 → google_cloud_db_context_engineering-0.6.0}/src/google_cloud_db_context_engineering.egg-info/requires.txt +0 -0
- {google_cloud_db_context_engineering-0.5.1 → google_cloud_db_context_engineering-0.6.0}/src/google_cloud_db_context_engineering.egg-info/top_level.txt +0 -0
{google_cloud_db_context_engineering-0.5.1 → google_cloud_db_context_engineering-0.6.0}/PKG-INFO
RENAMED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: google-cloud-db-context-engineering
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.6.0
|
|
4
4
|
Summary: A FastMCP server for generating natural language to SQL templates from database schemas.
|
|
5
5
|
Requires-Python: >=3.12
|
|
6
6
|
Description-Content-Type: text/markdown
|
|
@@ -92,8 +92,6 @@ To install the Context Engineering Agent via the Gemini CLI:
|
|
|
92
92
|
gemini extensions install https://github.com/GoogleCloudPlatform/db-context-enrichment
|
|
93
93
|
```
|
|
94
94
|
|
|
95
|
-
*Note: The extension requires a Gemini API key at installation or via environment variable `GEMINI_API_KEY`.*
|
|
96
|
-
|
|
97
95
|
### Optional: VSCode Integration
|
|
98
96
|
For an enhanced editing and diffing experience when reviewing context changes:
|
|
99
97
|
1. Install [VSCode](https://code.visualstudio.com/download).
|
|
@@ -110,23 +108,23 @@ The extension is designed to support the **Critical User Journeys (CUJs)** for c
|
|
|
110
108
|
4. **Iterate**: Apply the improved context and re-run evaluation to continuously improve quality.
|
|
111
109
|
5. **Final Validation** (Optional): Verify mutations against a separated test set to ensure generalization and prevent overfitting.
|
|
112
110
|
|
|
113
|
-
|
|
111
|
+
All workflows are exposed as Agent Skills. Activate them by asking the agent in natural language (e.g. "bootstrap the context for my database") or by invoking the skill by name. Start the Gemini CLI by running `gemini` in your workspace folder, or launch Claude Code with the plugin installed.
|
|
114
112
|
|
|
115
|
-
|
|
113
|
+
### Automated Iterative Optimization (Autoctx)
|
|
116
114
|
|
|
117
|
-
1. **Initialize (
|
|
118
|
-
2. **Generate Dataset (
|
|
119
|
-
3. **Bootstrap (
|
|
120
|
-
4. **Evaluate (
|
|
121
|
-
5. **Hill-Climb (
|
|
115
|
+
1. **Initialize (`autoctx-init`)**: Sets up your local workspace by creating an `autoctx/` directory. It checks for the presence of a valid `tools.yaml` configuration inside it. If missing, the agent will interactively prompt you for your database connection details and generate the file for you. It also creates the `state.md` file to track experiment progress and an `experiments/` directory, all within `autoctx/`.
|
|
116
|
+
2. **Generate Dataset (`autoctx-dataset-generation`)**: Rapidly creates or expands a baseline of evaluation questions (golden dataset). It asks you for sample queries or descriptions of what users might ask, and generates a JSON file with Natural Language Queries (NLQs) and Golden SQL statements.
|
|
117
|
+
3. **Bootstrap (`autoctx-bootstrap`)**: Generates an initial context set. It performs progressive schema discovery to understand your database structure and qualified tables. It then generates starting Templates and Facets based on the schema and any user-provided documentation or sample queries.
|
|
118
|
+
4. **Evaluate (`autoctx-evaluate`)**: Measures context effectiveness against your golden dataset. This step automatically generates all necessary Evalbench configuration files (`db_config.yaml`, `model_config.yaml`, `run_config.yaml`, `llmrater_config.yaml`) inside the experiment folder and runs the evaluation pipeline to produce accuracy scores and identify failure cases.
|
|
119
|
+
5. **Hill-Climb (`autoctx-hillclimb`)**: Performs gap analysis on failures identified in the evaluation step. It reads the failure cases, determines why the LLM failed to generate correct SQL, and proposes updates or new additions to the context set (Templates or Facets) to improve performance in the next iteration.
|
|
122
120
|
|
|
123
121
|
### Targeted Manual Generation
|
|
124
122
|
|
|
125
|
-
These are more basic workflows for context engineering to manually author specific context elements:
|
|
123
|
+
These are more basic workflows for context engineering to manually author specific context elements, each exposed via the `context-generation-guide` skill scoped to the relevant context type:
|
|
126
124
|
|
|
127
|
-
* **Generate Templates
|
|
128
|
-
* **Generate Facets
|
|
129
|
-
* **Generate Value Searches
|
|
125
|
+
* **Generate Templates**: Initiates a guided workflow where you provide a sample question and SQL, and the agent helps you parameterize and save it as a template.
|
|
126
|
+
* **Generate Facets**: Guides you to define a specific intent and the corresponding SQL snippet (e.g., filter condition) to save as a facet.
|
|
127
|
+
* **Generate Value Searches**: Helps you configure how the system searches for and matches specific values within a concept type (e.g., setting up exact match or trigram fuzzy search for product names).
|
|
130
128
|
|
|
131
129
|
## Development and Testing
|
|
132
130
|
|
{google_cloud_db_context_engineering-0.5.1 → google_cloud_db_context_engineering-0.6.0}/README.md
RENAMED
|
@@ -76,8 +76,6 @@ To install the Context Engineering Agent via the Gemini CLI:
|
|
|
76
76
|
gemini extensions install https://github.com/GoogleCloudPlatform/db-context-enrichment
|
|
77
77
|
```
|
|
78
78
|
|
|
79
|
-
*Note: The extension requires a Gemini API key at installation or via environment variable `GEMINI_API_KEY`.*
|
|
80
|
-
|
|
81
79
|
### Optional: VSCode Integration
|
|
82
80
|
For an enhanced editing and diffing experience when reviewing context changes:
|
|
83
81
|
1. Install [VSCode](https://code.visualstudio.com/download).
|
|
@@ -94,23 +92,23 @@ The extension is designed to support the **Critical User Journeys (CUJs)** for c
|
|
|
94
92
|
4. **Iterate**: Apply the improved context and re-run evaluation to continuously improve quality.
|
|
95
93
|
5. **Final Validation** (Optional): Verify mutations against a separated test set to ensure generalization and prevent overfitting.
|
|
96
94
|
|
|
97
|
-
|
|
95
|
+
All workflows are exposed as Agent Skills. Activate them by asking the agent in natural language (e.g. "bootstrap the context for my database") or by invoking the skill by name. Start the Gemini CLI by running `gemini` in your workspace folder, or launch Claude Code with the plugin installed.
|
|
98
96
|
|
|
99
|
-
|
|
97
|
+
### Automated Iterative Optimization (Autoctx)
|
|
100
98
|
|
|
101
|
-
1. **Initialize (
|
|
102
|
-
2. **Generate Dataset (
|
|
103
|
-
3. **Bootstrap (
|
|
104
|
-
4. **Evaluate (
|
|
105
|
-
5. **Hill-Climb (
|
|
99
|
+
1. **Initialize (`autoctx-init`)**: Sets up your local workspace by creating an `autoctx/` directory. It checks for the presence of a valid `tools.yaml` configuration inside it. If missing, the agent will interactively prompt you for your database connection details and generate the file for you. It also creates the `state.md` file to track experiment progress and an `experiments/` directory, all within `autoctx/`.
|
|
100
|
+
2. **Generate Dataset (`autoctx-dataset-generation`)**: Rapidly creates or expands a baseline of evaluation questions (golden dataset). It asks you for sample queries or descriptions of what users might ask, and generates a JSON file with Natural Language Queries (NLQs) and Golden SQL statements.
|
|
101
|
+
3. **Bootstrap (`autoctx-bootstrap`)**: Generates an initial context set. It performs progressive schema discovery to understand your database structure and qualified tables. It then generates starting Templates and Facets based on the schema and any user-provided documentation or sample queries.
|
|
102
|
+
4. **Evaluate (`autoctx-evaluate`)**: Measures context effectiveness against your golden dataset. This step automatically generates all necessary Evalbench configuration files (`db_config.yaml`, `model_config.yaml`, `run_config.yaml`, `llmrater_config.yaml`) inside the experiment folder and runs the evaluation pipeline to produce accuracy scores and identify failure cases.
|
|
103
|
+
5. **Hill-Climb (`autoctx-hillclimb`)**: Performs gap analysis on failures identified in the evaluation step. It reads the failure cases, determines why the LLM failed to generate correct SQL, and proposes updates or new additions to the context set (Templates or Facets) to improve performance in the next iteration.
|
|
106
104
|
|
|
107
105
|
### Targeted Manual Generation
|
|
108
106
|
|
|
109
|
-
These are more basic workflows for context engineering to manually author specific context elements:
|
|
107
|
+
These are more basic workflows for context engineering to manually author specific context elements, each exposed via the `context-generation-guide` skill scoped to the relevant context type:
|
|
110
108
|
|
|
111
|
-
* **Generate Templates
|
|
112
|
-
* **Generate Facets
|
|
113
|
-
* **Generate Value Searches
|
|
109
|
+
* **Generate Templates**: Initiates a guided workflow where you provide a sample question and SQL, and the agent helps you parameterize and save it as a template.
|
|
110
|
+
* **Generate Facets**: Guides you to define a specific intent and the corresponding SQL snippet (e.g., filter condition) to save as a facet.
|
|
111
|
+
* **Generate Value Searches**: Helps you configure how the system searches for and matches specific values within a concept type (e.g., setting up exact match or trigram fuzzy search for product names).
|
|
114
112
|
|
|
115
113
|
## Development and Testing
|
|
116
114
|
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "google-cloud-db-context-engineering"
|
|
3
|
-
version = "0.
|
|
3
|
+
version = "0.6.0"
|
|
4
4
|
description = "A FastMCP server for generating natural language to SQL templates from database schemas."
|
|
5
5
|
readme = "README.md"
|
|
6
6
|
requires-python = ">=3.12"
|
|
@@ -43,8 +43,8 @@ dev = [
|
|
|
43
43
|
]
|
|
44
44
|
|
|
45
45
|
[tool.db-context-engineering]
|
|
46
|
-
toolbox_version = "
|
|
47
|
-
evalbench_version = "1.
|
|
46
|
+
toolbox_version = "1.4.0"
|
|
47
|
+
evalbench_version = "1.9.0"
|
|
48
48
|
|
|
49
49
|
[tool.ruff]
|
|
50
50
|
line-length = 88
|
|
@@ -0,0 +1,197 @@
|
|
|
1
|
+
import json
|
|
2
|
+
|
|
3
|
+
from fastmcp import FastMCP
|
|
4
|
+
|
|
5
|
+
from google.cloud.db_context_enrichment.common import context_mutator
|
|
6
|
+
from google.cloud.db_context_enrichment.dataset import dataset_generator
|
|
7
|
+
from google.cloud.db_context_enrichment.evaluate import (
|
|
8
|
+
evaluate_generator,
|
|
9
|
+
result_reader,
|
|
10
|
+
)
|
|
11
|
+
|
|
12
|
+
mcp = FastMCP("Context Engineering Agent MCP")
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
@mcp.tool
|
|
16
|
+
async def generate_dataset(
|
|
17
|
+
dataset_entries_json: str,
|
|
18
|
+
output_file_path: str,
|
|
19
|
+
) -> str:
|
|
20
|
+
"""
|
|
21
|
+
Validates a list of evaluation dataset entries and saves them to a JSON file.
|
|
22
|
+
|
|
23
|
+
Args:
|
|
24
|
+
dataset_entries_json: A JSON string representing a list of dataset items.
|
|
25
|
+
Each item should have "id", "database", "nlq", and "golden_sql" keys.
|
|
26
|
+
Example: '[{"id": "eval_001", "database": "my_db", "nlq": "Count users", "golden_sql": "SELECT COUNT(*) FROM users"}]'
|
|
27
|
+
output_file_path: The absolute path where the dataset JSON file should be saved.
|
|
28
|
+
|
|
29
|
+
Returns:
|
|
30
|
+
The absolute file path where the dataset was saved.
|
|
31
|
+
"""
|
|
32
|
+
return await dataset_generator.generate_dataset(
|
|
33
|
+
dataset_entries_json, output_file_path
|
|
34
|
+
)
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
@mcp.tool
|
|
38
|
+
def generate_evalbench_configs(
|
|
39
|
+
experiment_name: str,
|
|
40
|
+
dataset_path: str,
|
|
41
|
+
context_set_id: str,
|
|
42
|
+
toolbox_config_path: str,
|
|
43
|
+
toolbox_source_name: str,
|
|
44
|
+
) -> str:
|
|
45
|
+
"""
|
|
46
|
+
Generates Evalbench YAML configurations and converts the user-facing golden dataset to be compatible for evaluation, saving all files directly to disk.
|
|
47
|
+
|
|
48
|
+
This tool writes the following files inside `experiments/<experiment_name>/eval_configs/`:
|
|
49
|
+
- `db_config.yaml`
|
|
50
|
+
- `model_config.yaml`
|
|
51
|
+
- `run_config.yaml`
|
|
52
|
+
- `llmrater_config.yaml`
|
|
53
|
+
- `golden_queries.json` (converted to EvalBench internal format)
|
|
54
|
+
|
|
55
|
+
Args:
|
|
56
|
+
experiment_name: The name of the target experiment folder.
|
|
57
|
+
dataset_path: The absolute path to the golden dataset file in the simplified user-facing format (JSON list of objects with keys: "id", "database", "nlq", "golden_sql").
|
|
58
|
+
context_set_id: The specific context_set_id inside the experiment.
|
|
59
|
+
toolbox_config_path: The absolute path to the tools.yaml configuration file.
|
|
60
|
+
toolbox_source_name: The name of the database source to use inside tools.yaml. The underlying source block must use a supported 'type' (cloud-sql-postgres, cloud-sql-mysql, spanner, alloydb-postgres).
|
|
61
|
+
|
|
62
|
+
Returns:
|
|
63
|
+
A message indicating that the configuration files were successfully created.
|
|
64
|
+
"""
|
|
65
|
+
evaluate_generator.generate_evalbench_configs(
|
|
66
|
+
experiment_name,
|
|
67
|
+
dataset_path,
|
|
68
|
+
context_set_id,
|
|
69
|
+
toolbox_config_path,
|
|
70
|
+
toolbox_source_name,
|
|
71
|
+
)
|
|
72
|
+
return f"Successfully generated all configs for evaluation in experiments/{experiment_name}/eval_configs/"
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
@mcp.tool
|
|
76
|
+
def generate_upload_url(
|
|
77
|
+
db_engine: str,
|
|
78
|
+
project_id: str,
|
|
79
|
+
location: str | None = None,
|
|
80
|
+
cluster_id: str | None = None,
|
|
81
|
+
instance_id: str | None = None,
|
|
82
|
+
database_id: str | None = None,
|
|
83
|
+
) -> str:
|
|
84
|
+
"""
|
|
85
|
+
Generates a URL for uploading the template file based on the database engine.
|
|
86
|
+
|
|
87
|
+
Args:
|
|
88
|
+
db_engine: The database engine. Accepted values are 'alloydb',
|
|
89
|
+
'cloudsql', or 'spanner'. This can be derived from the 'kind'
|
|
90
|
+
field in the tools.yaml file. For example, 'alloydb-postgres'
|
|
91
|
+
becomes 'alloydb', and 'cloud-sql-postgres' becomes 'cloudsql'.
|
|
92
|
+
project_id: The Google Cloud project ID.
|
|
93
|
+
location: The location of the AlloyDB cluster.
|
|
94
|
+
cluster_id: The ID of the AlloyDB cluster.
|
|
95
|
+
instance_id: The ID of the Cloud SQL or Spanner instance.
|
|
96
|
+
database_id: The ID of the Spanner database.
|
|
97
|
+
|
|
98
|
+
Returns:
|
|
99
|
+
The generated URL as a string, or an error message if the source kind is invalid.
|
|
100
|
+
"""
|
|
101
|
+
if db_engine == "alloydb":
|
|
102
|
+
if location and cluster_id and project_id:
|
|
103
|
+
return f"https://console.cloud.google.com/alloydb/locations/{location}/clusters/{cluster_id}/studio?project={project_id}"
|
|
104
|
+
else:
|
|
105
|
+
return "Error: Missing location, cluster_id, or project_id for alloydb."
|
|
106
|
+
elif db_engine == "cloudsql":
|
|
107
|
+
if instance_id and project_id:
|
|
108
|
+
return f"https://console.cloud.google.com/sql/instances/{instance_id}/studio?project={project_id}"
|
|
109
|
+
else:
|
|
110
|
+
return "Error: Missing instance_id or project_id for cloudsql."
|
|
111
|
+
elif db_engine == "spanner":
|
|
112
|
+
if instance_id and database_id and project_id:
|
|
113
|
+
return f"https://console.cloud.google.com/spanner/instances/{instance_id}/databases/{database_id}/details/query?project={project_id}"
|
|
114
|
+
else:
|
|
115
|
+
return "Error: Missing instance_id, database_id, or project_id for spanner."
|
|
116
|
+
else:
|
|
117
|
+
return "Error: Invalid db_engine. Must be one of 'alloydb', 'cloudsql', or 'spanner'."
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
@mcp.tool
|
|
121
|
+
def mutate_context_set(
|
|
122
|
+
file_path: str,
|
|
123
|
+
mutations_json: str,
|
|
124
|
+
) -> str:
|
|
125
|
+
"""
|
|
126
|
+
Apply structural mutations to an existing ContextSet JSON file.
|
|
127
|
+
|
|
128
|
+
Parameters:
|
|
129
|
+
- file_path (str): The absolute path to the ContextSet file.
|
|
130
|
+
- mutations_json (str): A JSON string representing a list of mutations.
|
|
131
|
+
Each mutation must contain:
|
|
132
|
+
- 'operation': "add", "delete", or "update"
|
|
133
|
+
- 'type': "template", "facet", or "value_search"
|
|
134
|
+
- 'identifier' (dict): Required for "delete" and "update" to find the target item (e.g., {"nl_query": "What are all users?"}).
|
|
135
|
+
- 'value' (dict): Required for "add" and "update".
|
|
136
|
+
- For "add": Must be the FULL item body. Follow the formatting guidance in the `context-generation-guide` skill to produce well-formed content.
|
|
137
|
+
- For "update": Can be a PARTIAL body containing only the fields to change (it will be merged with the existing item).
|
|
138
|
+
|
|
139
|
+
Example 'mutations_json':
|
|
140
|
+
'[
|
|
141
|
+
{
|
|
142
|
+
"operation": "add",
|
|
143
|
+
"type": "template",
|
|
144
|
+
"value": {
|
|
145
|
+
"nl_query": "How many users registered in 2023?",
|
|
146
|
+
"sql": "SELECT count(*) FROM users WHERE year = 2023",
|
|
147
|
+
"intent": "Count users registered in 2023",
|
|
148
|
+
"manifest": "Count users registered in a given year",
|
|
149
|
+
"parameterized": {
|
|
150
|
+
"parameterized_sql": "SELECT count(*) FROM users WHERE year = $1",
|
|
151
|
+
"parameterized_intent": "Count users registered in $1"
|
|
152
|
+
}
|
|
153
|
+
}
|
|
154
|
+
},
|
|
155
|
+
{
|
|
156
|
+
"operation": "delete",
|
|
157
|
+
"type": "facet",
|
|
158
|
+
"identifier": {"intent": "high price"}
|
|
159
|
+
},
|
|
160
|
+
{
|
|
161
|
+
"operation": "update",
|
|
162
|
+
"type": "facet",
|
|
163
|
+
"identifier": {"intent": "high price"},
|
|
164
|
+
"value": {"sql_snippet": "price > 2000", "intent": "very high price"}
|
|
165
|
+
}
|
|
166
|
+
]'
|
|
167
|
+
"""
|
|
168
|
+
try:
|
|
169
|
+
mutations_data = json.loads(mutations_json)
|
|
170
|
+
if not isinstance(mutations_data, list):
|
|
171
|
+
return "Error applying mutations: mutations_json must be a JSON list."
|
|
172
|
+
mutations = [context_mutator.Mutation(**mut) for mut in mutations_data]
|
|
173
|
+
context_mutator.mutate_context_set(file_path, mutations)
|
|
174
|
+
return f"Successfully applied {len(mutations)} mutations to {file_path}"
|
|
175
|
+
except Exception as e:
|
|
176
|
+
return f"Error applying mutations: {str(e)}"
|
|
177
|
+
|
|
178
|
+
|
|
179
|
+
@mcp.tool
|
|
180
|
+
async def read_evaluation_result(
|
|
181
|
+
run_folder_path: str, offset: int = 0, batch_size: int = 10
|
|
182
|
+
) -> str:
|
|
183
|
+
"""Reads evaluation results from a folder and produces a markdown summary.
|
|
184
|
+
|
|
185
|
+
Args:
|
|
186
|
+
run_folder_path: The absolute path to the evaluation run result folder, which ends with the eval run job id.
|
|
187
|
+
offset: Offset to start reading failure cases from (default: 0).
|
|
188
|
+
batch_size: Number of failure cases to show in the report (default: 10).
|
|
189
|
+
|
|
190
|
+
Returns:
|
|
191
|
+
A string in markdown format containing the summary and failure cases.
|
|
192
|
+
"""
|
|
193
|
+
return result_reader.read_eval_results(run_folder_path, offset, batch_size)
|
|
194
|
+
|
|
195
|
+
|
|
196
|
+
if __name__ == "__main__":
|
|
197
|
+
mcp.run() # Uses STDIO transport by default
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: google-cloud-db-context-engineering
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.6.0
|
|
4
4
|
Summary: A FastMCP server for generating natural language to SQL templates from database schemas.
|
|
5
5
|
Requires-Python: >=3.12
|
|
6
6
|
Description-Content-Type: text/markdown
|
|
@@ -92,8 +92,6 @@ To install the Context Engineering Agent via the Gemini CLI:
|
|
|
92
92
|
gemini extensions install https://github.com/GoogleCloudPlatform/db-context-enrichment
|
|
93
93
|
```
|
|
94
94
|
|
|
95
|
-
*Note: The extension requires a Gemini API key at installation or via environment variable `GEMINI_API_KEY`.*
|
|
96
|
-
|
|
97
95
|
### Optional: VSCode Integration
|
|
98
96
|
For an enhanced editing and diffing experience when reviewing context changes:
|
|
99
97
|
1. Install [VSCode](https://code.visualstudio.com/download).
|
|
@@ -110,23 +108,23 @@ The extension is designed to support the **Critical User Journeys (CUJs)** for c
|
|
|
110
108
|
4. **Iterate**: Apply the improved context and re-run evaluation to continuously improve quality.
|
|
111
109
|
5. **Final Validation** (Optional): Verify mutations against a separated test set to ensure generalization and prevent overfitting.
|
|
112
110
|
|
|
113
|
-
|
|
111
|
+
All workflows are exposed as Agent Skills. Activate them by asking the agent in natural language (e.g. "bootstrap the context for my database") or by invoking the skill by name. Start the Gemini CLI by running `gemini` in your workspace folder, or launch Claude Code with the plugin installed.
|
|
114
112
|
|
|
115
|
-
|
|
113
|
+
### Automated Iterative Optimization (Autoctx)
|
|
116
114
|
|
|
117
|
-
1. **Initialize (
|
|
118
|
-
2. **Generate Dataset (
|
|
119
|
-
3. **Bootstrap (
|
|
120
|
-
4. **Evaluate (
|
|
121
|
-
5. **Hill-Climb (
|
|
115
|
+
1. **Initialize (`autoctx-init`)**: Sets up your local workspace by creating an `autoctx/` directory. It checks for the presence of a valid `tools.yaml` configuration inside it. If missing, the agent will interactively prompt you for your database connection details and generate the file for you. It also creates the `state.md` file to track experiment progress and an `experiments/` directory, all within `autoctx/`.
|
|
116
|
+
2. **Generate Dataset (`autoctx-dataset-generation`)**: Rapidly creates or expands a baseline of evaluation questions (golden dataset). It asks you for sample queries or descriptions of what users might ask, and generates a JSON file with Natural Language Queries (NLQs) and Golden SQL statements.
|
|
117
|
+
3. **Bootstrap (`autoctx-bootstrap`)**: Generates an initial context set. It performs progressive schema discovery to understand your database structure and qualified tables. It then generates starting Templates and Facets based on the schema and any user-provided documentation or sample queries.
|
|
118
|
+
4. **Evaluate (`autoctx-evaluate`)**: Measures context effectiveness against your golden dataset. This step automatically generates all necessary Evalbench configuration files (`db_config.yaml`, `model_config.yaml`, `run_config.yaml`, `llmrater_config.yaml`) inside the experiment folder and runs the evaluation pipeline to produce accuracy scores and identify failure cases.
|
|
119
|
+
5. **Hill-Climb (`autoctx-hillclimb`)**: Performs gap analysis on failures identified in the evaluation step. It reads the failure cases, determines why the LLM failed to generate correct SQL, and proposes updates or new additions to the context set (Templates or Facets) to improve performance in the next iteration.
|
|
122
120
|
|
|
123
121
|
### Targeted Manual Generation
|
|
124
122
|
|
|
125
|
-
These are more basic workflows for context engineering to manually author specific context elements:
|
|
123
|
+
These are more basic workflows for context engineering to manually author specific context elements, each exposed via the `context-generation-guide` skill scoped to the relevant context type:
|
|
126
124
|
|
|
127
|
-
* **Generate Templates
|
|
128
|
-
* **Generate Facets
|
|
129
|
-
* **Generate Value Searches
|
|
125
|
+
* **Generate Templates**: Initiates a guided workflow where you provide a sample question and SQL, and the agent helps you parameterize and save it as a template.
|
|
126
|
+
* **Generate Facets**: Guides you to define a specific intent and the corresponding SQL snippet (e.g., filter condition) to save as a facet.
|
|
127
|
+
* **Generate Value Searches**: Helps you configure how the system searches for and matches specific values within a concept type (e.g., setting up exact match or trigram fuzzy search for product names).
|
|
130
128
|
|
|
131
129
|
## Development and Testing
|
|
132
130
|
|
|
@@ -3,12 +3,9 @@ README.md
|
|
|
3
3
|
pyproject.toml
|
|
4
4
|
src/google/cloud/db_context_enrichment/__init__.py
|
|
5
5
|
src/google/cloud/db_context_enrichment/main.py
|
|
6
|
-
src/google/cloud/db_context_enrichment/bootstrap/__init__.py
|
|
7
|
-
src/google/cloud/db_context_enrichment/bootstrap/bootstrap_generator.py
|
|
8
6
|
src/google/cloud/db_context_enrichment/common/__init__.py
|
|
9
7
|
src/google/cloud/db_context_enrichment/common/config.py
|
|
10
8
|
src/google/cloud/db_context_enrichment/common/context_mutator.py
|
|
11
|
-
src/google/cloud/db_context_enrichment/common/parameterizer.py
|
|
12
9
|
src/google/cloud/db_context_enrichment/dataset/__init__.py
|
|
13
10
|
src/google/cloud/db_context_enrichment/dataset/dataset_generator.py
|
|
14
11
|
src/google/cloud/db_context_enrichment/evaluate/__init__.py
|
|
@@ -20,19 +17,8 @@ src/google/cloud/db_context_enrichment/evaluate/db_generators/base.py
|
|
|
20
17
|
src/google/cloud/db_context_enrichment/evaluate/db_generators/mysql.py
|
|
21
18
|
src/google/cloud/db_context_enrichment/evaluate/db_generators/postgres.py
|
|
22
19
|
src/google/cloud/db_context_enrichment/evaluate/db_generators/spanner.py
|
|
23
|
-
src/google/cloud/db_context_enrichment/facet/__init__.py
|
|
24
|
-
src/google/cloud/db_context_enrichment/facet/facet_generator.py
|
|
25
20
|
src/google/cloud/db_context_enrichment/model/__init__.py
|
|
26
21
|
src/google/cloud/db_context_enrichment/model/context.py
|
|
27
|
-
src/google/cloud/db_context_enrichment/prompts/__init__.py
|
|
28
|
-
src/google/cloud/db_context_enrichment/prompts/targeted_facets.py
|
|
29
|
-
src/google/cloud/db_context_enrichment/prompts/targeted_templates.py
|
|
30
|
-
src/google/cloud/db_context_enrichment/prompts/targeted_value_search.py
|
|
31
|
-
src/google/cloud/db_context_enrichment/template/__init__.py
|
|
32
|
-
src/google/cloud/db_context_enrichment/template/template_generator.py
|
|
33
|
-
src/google/cloud/db_context_enrichment/value_search/__init__.py
|
|
34
|
-
src/google/cloud/db_context_enrichment/value_search/generator.py
|
|
35
|
-
src/google/cloud/db_context_enrichment/value_search/match_templates.py
|
|
36
22
|
src/google_cloud_db_context_engineering.egg-info/PKG-INFO
|
|
37
23
|
src/google_cloud_db_context_engineering.egg-info/SOURCES.txt
|
|
38
24
|
src/google_cloud_db_context_engineering.egg-info/dependency_links.txt
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
# Initialization script for bootstrap module
|
|
@@ -1,74 +0,0 @@
|
|
|
1
|
-
import json
|
|
2
|
-
|
|
3
|
-
from pydantic import ValidationError
|
|
4
|
-
|
|
5
|
-
from google.cloud.db_context_enrichment.common.context_mutator import (
|
|
6
|
-
Mutation,
|
|
7
|
-
mutate_context_set,
|
|
8
|
-
)
|
|
9
|
-
from google.cloud.db_context_enrichment.facet import facet_generator
|
|
10
|
-
from google.cloud.db_context_enrichment.model import context
|
|
11
|
-
from google.cloud.db_context_enrichment.template import template_generator
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
async def generate_context(
|
|
15
|
-
output_file_path: str,
|
|
16
|
-
sql_dialect: str,
|
|
17
|
-
template_inputs_json: str | None = None,
|
|
18
|
-
facet_inputs_json: str | None = None,
|
|
19
|
-
) -> str:
|
|
20
|
-
"""
|
|
21
|
-
Core logic for generating a single unified ContextSet from key information and saving it to a file.
|
|
22
|
-
"""
|
|
23
|
-
final_templates = None
|
|
24
|
-
final_facets = None
|
|
25
|
-
|
|
26
|
-
if template_inputs_json:
|
|
27
|
-
res_str = await template_generator.generate_templates(
|
|
28
|
-
template_inputs_json, sql_dialect
|
|
29
|
-
)
|
|
30
|
-
if '"error":' in res_str:
|
|
31
|
-
raise RuntimeError(f"Error generating templates: {res_str}")
|
|
32
|
-
try:
|
|
33
|
-
res_dict = json.loads(res_str)
|
|
34
|
-
final_templates = [
|
|
35
|
-
context.Template(**t) for t in res_dict.get("templates", [])
|
|
36
|
-
]
|
|
37
|
-
except (json.JSONDecodeError, ValidationError) as e:
|
|
38
|
-
raise ValueError(f"Error parsing generated templates: {e}") from e
|
|
39
|
-
|
|
40
|
-
if facet_inputs_json:
|
|
41
|
-
res_str = await facet_generator.generate_facets(facet_inputs_json, sql_dialect)
|
|
42
|
-
if '"error":' in res_str:
|
|
43
|
-
raise RuntimeError(f"Error generating facets: {res_str}")
|
|
44
|
-
try:
|
|
45
|
-
res_dict = json.loads(res_str)
|
|
46
|
-
final_facets = [context.Facet(**f) for f in res_dict.get("facets", [])]
|
|
47
|
-
except (json.JSONDecodeError, ValidationError) as e:
|
|
48
|
-
raise ValueError(f"Error parsing generated facets: {e}") from e
|
|
49
|
-
|
|
50
|
-
mutations: list[Mutation] = []
|
|
51
|
-
|
|
52
|
-
if final_templates:
|
|
53
|
-
for t in final_templates:
|
|
54
|
-
mutations.append(
|
|
55
|
-
Mutation(
|
|
56
|
-
operation="add",
|
|
57
|
-
type="template",
|
|
58
|
-
value=t.model_dump(exclude_none=True),
|
|
59
|
-
)
|
|
60
|
-
)
|
|
61
|
-
|
|
62
|
-
if final_facets:
|
|
63
|
-
for f in final_facets:
|
|
64
|
-
mutations.append(
|
|
65
|
-
Mutation(
|
|
66
|
-
operation="add", type="facet", value=f.model_dump(exclude_none=True)
|
|
67
|
-
)
|
|
68
|
-
)
|
|
69
|
-
|
|
70
|
-
if not mutations:
|
|
71
|
-
raise ValueError("No templates or facets were generated to save.")
|
|
72
|
-
|
|
73
|
-
mutate_context_set(output_file_path, mutations)
|
|
74
|
-
return output_file_path
|