dbt-cortex-agent 0.0.3__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (40) hide show
  1. dbt_cortex_agent/__init__.py +5 -0
  2. dbt_cortex_agent/artifacts.py +26 -0
  3. dbt_cortex_agent/cli.py +90 -0
  4. dbt_cortex_agent/commands/__init__.py +1 -0
  5. dbt_cortex_agent/commands/agent.py +115 -0
  6. dbt_cortex_agent/commands/bootstrap.py +92 -0
  7. dbt_cortex_agent/commands/common.py +62 -0
  8. dbt_cortex_agent/commands/eval.py +143 -0
  9. dbt_cortex_agent/commands/manifest.py +48 -0
  10. dbt_cortex_agent/commands/skill.py +129 -0
  11. dbt_cortex_agent/config.py +88 -0
  12. dbt_cortex_agent/dbt_runner.py +93 -0
  13. dbt_cortex_agent/doctor.py +204 -0
  14. dbt_cortex_agent/eval/__init__.py +0 -0
  15. dbt_cortex_agent/eval/baseline.py +44 -0
  16. dbt_cortex_agent/eval/compare.py +68 -0
  17. dbt_cortex_agent/eval/dataset.py +128 -0
  18. dbt_cortex_agent/eval/gate.py +31 -0
  19. dbt_cortex_agent/eval/lifecycle.py +567 -0
  20. dbt_cortex_agent/eval/results.py +195 -0
  21. dbt_cortex_agent/execution_context.py +136 -0
  22. dbt_cortex_agent/identifiers.py +46 -0
  23. dbt_cortex_agent/init.py +322 -0
  24. dbt_cortex_agent/invoke.py +140 -0
  25. dbt_cortex_agent/manifest.py +285 -0
  26. dbt_cortex_agent/skills.py +181 -0
  27. dbt_cortex_agent/snow.py +11 -0
  28. dbt_cortex_agent/starters/orders/models/agents/orders_assistant/agent.yml +7 -0
  29. dbt_cortex_agent/starters/orders/models/agents/orders_assistant/evals/core.yml +54 -0
  30. dbt_cortex_agent/starters/orders/models/agents/orders_assistant/evals/orders_assistant_core.sql +50 -0
  31. dbt_cortex_agent/starters/orders/models/agents/orders_assistant/orders_assistant.sql +54 -0
  32. dbt_cortex_agent/starters/orders/models/semantic/sem_orders.sql +40 -0
  33. dbt_cortex_agent/starters/orders/seeds/orders.csv +4 -0
  34. dbt_cortex_agent/starters/orders/seeds/orders.yml +19 -0
  35. dbt_cortex_agent-0.0.3.dist-info/METADATA +196 -0
  36. dbt_cortex_agent-0.0.3.dist-info/RECORD +40 -0
  37. dbt_cortex_agent-0.0.3.dist-info/WHEEL +5 -0
  38. dbt_cortex_agent-0.0.3.dist-info/entry_points.txt +2 -0
  39. dbt_cortex_agent-0.0.3.dist-info/licenses/LICENSE +201 -0
  40. dbt_cortex_agent-0.0.3.dist-info/top_level.txt +1 -0
@@ -0,0 +1,285 @@
1
+ from __future__ import annotations
2
+
3
+ import json
4
+ from dataclasses import dataclass
5
+ from pathlib import Path
6
+ from typing import Any
7
+
8
+ from .identifiers import fqn, identifier, stage_path
9
+
10
+
11
+ SUPPORTED_MANIFEST_SCHEMA_VERSIONS = {"v12"}
12
+
13
+
14
+ @dataclass(frozen=True)
15
+ class SkillDeclaration:
16
+ agent_name: str
17
+ skill_name: str
18
+ source_type: str
19
+ stage_path: str
20
+ local_dir: Path
21
+
22
+
23
+ @dataclass(frozen=True)
24
+ class EvalDeclaration:
25
+ model_name: str
26
+ table_fqn: str
27
+ meta: dict[str, Any]
28
+
29
+
30
+ def agent_database(agent: dict[str, Any]) -> str:
31
+ return identifier(str(agent.get("database") or ""), f"database for {agent.get('name')}")
32
+
33
+
34
+ def agent_schema(agent: dict[str, Any]) -> str:
35
+ return identifier(str(agent.get("schema") or ""), f"schema for {agent.get('name')}")
36
+
37
+
38
+ def selected_agent_databases(agents: list[dict[str, Any]]) -> tuple[str, ...]:
39
+ return tuple(sorted({agent_database(agent) for agent in agents}))
40
+
41
+
42
+ def assert_config_database(
43
+ manifest: dict[str, Any], database: str | None, agent_names: list[str] | None = None
44
+ ) -> str:
45
+ """Compatibility assertion scoped to selected Agents, never unrelated models."""
46
+ if not database:
47
+ raise ValueError("Operation requires explicit --database or SNOWFLAKE_DATABASE")
48
+ configured = identifier(database, "configured database")
49
+ resolved = selected_agent_databases(select_agents(manifest, agent_names))
50
+ if resolved != (configured,):
51
+ raise ValueError(
52
+ f"Configured database {configured!r} does not match selected Agent databases "
53
+ f"{', '.join(resolved) or 'none'}"
54
+ )
55
+ return configured
56
+
57
+
58
+ def assert_resource_databases_allowed(
59
+ databases: list[str] | tuple[str, ...] | set[str], allowed_databases: list[str]
60
+ ) -> tuple[str, ...]:
61
+ resolved = tuple(sorted({identifier(value, "resource database") for value in databases}))
62
+ allowed = {identifier(value, "allowed database") for value in allowed_databases}
63
+ missing = [value for value in resolved if value not in allowed]
64
+ if missing:
65
+ raise ValueError(
66
+ "Refusing operation for resource databases not in the allowlist: "
67
+ f"{', '.join(missing)}; allowed databases: {', '.join(sorted(allowed)) or 'none'}"
68
+ )
69
+ return resolved
70
+
71
+
72
+ def load_manifest(path: str | Path) -> dict[str, Any]:
73
+ manifest_path = Path(path)
74
+ if not manifest_path.is_file():
75
+ raise FileNotFoundError(
76
+ f"dbt manifest not found: {manifest_path}. Run `dbt parse` for this project first."
77
+ )
78
+ try:
79
+ manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
80
+ except json.JSONDecodeError as exc:
81
+ raise ValueError(f"Invalid JSON in dbt manifest {manifest_path}: {exc}") from exc
82
+ if not isinstance(manifest, dict):
83
+ raise ValueError(f"dbt manifest {manifest_path} must contain a JSON object")
84
+ validate_manifest(manifest, manifest_path)
85
+ return manifest
86
+
87
+
88
+ def validate_manifest(manifest: dict[str, Any], manifest_path: str | Path | None = None) -> None:
89
+ metadata = manifest.get("metadata")
90
+ schema_version = metadata.get("dbt_schema_version") if isinstance(metadata, dict) else None
91
+ if not schema_version:
92
+ location = f" in {manifest_path}" if manifest_path else ""
93
+ raise ValueError(f"dbt manifest{location} is missing metadata.dbt_schema_version")
94
+ version = str(schema_version).rstrip("/").split("/")[-1].removesuffix(".json")
95
+ if version not in SUPPORTED_MANIFEST_SCHEMA_VERSIONS:
96
+ supported = ", ".join(sorted(SUPPORTED_MANIFEST_SCHEMA_VERSIONS))
97
+ raise ValueError(
98
+ f"Unsupported dbt manifest schema version {schema_version!r}; supported versions: "
99
+ f"{supported}. Run a supported dbt version or upgrade dbt-cortex-agent."
100
+ )
101
+
102
+
103
+ def _meta(obj: dict[str, Any]) -> dict[str, Any]:
104
+ direct = obj.get("meta")
105
+ if isinstance(direct, dict) and direct:
106
+ return direct
107
+ config = obj.get("config")
108
+ configured = config.get("meta") if isinstance(config, dict) else None
109
+ return configured if isinstance(configured, dict) else {}
110
+
111
+
112
+ def _relation_fqn(node: dict[str, Any]) -> str:
113
+ database = str(node.get("database") or "")
114
+ schema = str(node.get("schema") or "")
115
+ relation = str(node.get("alias") or node.get("name") or "")
116
+ if not all((database, schema, relation)):
117
+ raise ValueError(f"Eval model is missing database, schema, or name: {node.get('unique_id')}")
118
+ return fqn(f"{database}.{schema}.{relation}", "eval model")
119
+
120
+
121
+ def _model_agent_spec(node: dict[str, Any]) -> dict[str, Any]:
122
+ compiled = node.get("compiled_code")
123
+ if not isinstance(compiled, str) or not compiled.strip():
124
+ return {}
125
+ try:
126
+ value = json.loads(compiled)
127
+ except json.JSONDecodeError:
128
+ return {}
129
+ return value if isinstance(value, dict) else {}
130
+
131
+
132
+ def _stage_suffix(stage_path: str) -> Path:
133
+ _, suffix = stage_path_parts(stage_path)
134
+ return Path(suffix)
135
+
136
+
137
+ def stage_path_parts(value: str) -> tuple[str, str]:
138
+ return stage_path(value, "skill stage path")
139
+
140
+
141
+ def local_skill_dir(project_dir: str | Path, stage_path: str) -> Path:
142
+ project_path = Path(project_dir)
143
+ suffix = _stage_suffix(stage_path)
144
+ parts = suffix.parts
145
+ if parts and parts[0] == "agents":
146
+ if len(parts) < 3:
147
+ raise ValueError(
148
+ f"Private skill stage path must be agents/<agent>/<name>: {stage_path!r}"
149
+ )
150
+ return project_path / "models" / "agents" / parts[1] / "skills" / Path(*parts[2:])
151
+ return project_path / "skills" / suffix
152
+
153
+
154
+ def cortex_agents(manifest: dict[str, Any]) -> list[dict[str, Any]]:
155
+ agents: list[dict[str, Any]] = []
156
+ nodes = manifest.get("nodes") or {}
157
+ for node in nodes.values():
158
+ config = node.get("config") if isinstance(node, dict) else None
159
+ if not isinstance(config, dict) or config.get("materialized") != "cortex_agent":
160
+ continue
161
+ agent_meta = _meta(node).get("cortex_agent", {})
162
+ if agent_meta is None:
163
+ agent_meta = {}
164
+ if not isinstance(agent_meta, dict):
165
+ raise ValueError(
166
+ f"cortex_agent model {node.get('unique_id')!r} meta.cortex_agent must be a mapping"
167
+ )
168
+ if agent_meta.get("enabled") is False:
169
+ continue
170
+ name = node.get("name")
171
+ if not name:
172
+ raise ValueError("Enabled cortex_agent model is missing a name")
173
+ database = identifier(str(node.get("database") or ""), "cortex_agent model database")
174
+ schema = identifier(str(node.get("schema") or ""), "cortex_agent model schema")
175
+ physical_name = identifier(
176
+ str(node.get("alias") or name), f"physical Agent for {name}"
177
+ )
178
+ normalized_meta = {
179
+ **agent_meta,
180
+ "compiled_spec": _model_agent_spec(node),
181
+ "snowflake_name": physical_name,
182
+ }
183
+ agents.append(
184
+ {
185
+ "name": str(name),
186
+ "meta": normalized_meta,
187
+ "resource_type": "model",
188
+ "unique_id": node.get("unique_id"),
189
+ "database": database,
190
+ "schema": schema,
191
+ "object_name": physical_name,
192
+ "physical_fqn": f"{database}.{schema}.{physical_name}",
193
+ }
194
+ )
195
+ return sorted(agents, key=lambda item: item["name"])
196
+
197
+
198
+ def select_agents(
199
+ manifest: dict[str, Any], names: list[str] | tuple[str, ...] | None = None
200
+ ) -> list[dict[str, Any]]:
201
+ agents = cortex_agents(manifest)
202
+ by_name: dict[str, list[dict[str, Any]]] = {}
203
+ for agent in agents:
204
+ by_name.setdefault(agent["name"], []).append(agent)
205
+ duplicates = sorted(name for name, values in by_name.items() if len(values) > 1)
206
+ if duplicates:
207
+ raise ValueError(
208
+ "Enabled cortex_agent model names must be unique: "
209
+ f"{', '.join(duplicates)}"
210
+ )
211
+ physical: dict[str, list[str]] = {}
212
+ for agent in agents:
213
+ physical_name = agent.get("physical_fqn")
214
+ if physical_name:
215
+ physical.setdefault(str(physical_name).upper(), []).append(agent["name"])
216
+ duplicate_physical = sorted(
217
+ name for name, logical_names in physical.items() if len(logical_names) > 1
218
+ )
219
+ if duplicate_physical:
220
+ raise ValueError(
221
+ "Enabled cortex_agent physical identities must be unique: "
222
+ f"{', '.join(duplicate_physical)}"
223
+ )
224
+ if not names:
225
+ return agents
226
+ requested = list(dict.fromkeys(names))
227
+ missing = sorted(set(requested) - set(by_name))
228
+ if missing:
229
+ available = ", ".join(sorted(by_name)) or "none"
230
+ raise ValueError(
231
+ f"Unknown enabled cortex_agent selection: {', '.join(missing)}; available: {available}"
232
+ )
233
+ return [by_name[name][0] for name in requested]
234
+
235
+
236
+ def skill_declarations(
237
+ manifest: dict[str, Any], project_dir: str | Path, agent_names: list[str] | None = None
238
+ ) -> list[SkillDeclaration]:
239
+ declarations: list[SkillDeclaration] = []
240
+ for agent in select_agents(manifest, agent_names):
241
+ configured = agent["meta"].get("skills") or []
242
+ spec_skills = agent["meta"].get("compiled_spec", {}).get("skills") or []
243
+ skills = configured or spec_skills
244
+ for skill in skills:
245
+ source = skill.get("source") or {}
246
+ if str(source.get("type", "")).lower() != "stage":
247
+ continue
248
+ stage_path = source.get("path")
249
+ if not stage_path or not skill.get("name"):
250
+ raise ValueError(f"Agent {agent['name']!r} has an incomplete stage skill declaration")
251
+ declarations.append(
252
+ SkillDeclaration(
253
+ agent_name=agent["name"],
254
+ skill_name=skill["name"],
255
+ source_type=source["type"],
256
+ stage_path=stage_path,
257
+ local_dir=local_skill_dir(project_dir, stage_path),
258
+ )
259
+ )
260
+ return declarations
261
+
262
+
263
+ def cortex_evals(manifest: dict[str, Any]) -> list[EvalDeclaration]:
264
+ declarations: list[EvalDeclaration] = []
265
+ nodes = manifest.get("nodes") or {}
266
+ for node in nodes.values():
267
+ eval_meta = _meta(node).get("cortex_eval")
268
+ if isinstance(eval_meta, dict) and eval_meta and eval_meta.get("enabled") is not False:
269
+ declarations.append(
270
+ EvalDeclaration(
271
+ model_name=node["name"], table_fqn=_relation_fqn(node), meta=eval_meta
272
+ )
273
+ )
274
+ return sorted(declarations, key=lambda item: item.model_name)
275
+
276
+
277
+ def physical_agent_name(agent: dict[str, Any], target: str | None) -> str:
278
+ return identifier(
279
+ agent["meta"].get("snowflake_name"),
280
+ f"physical Agent for {agent['name']}",
281
+ )
282
+
283
+
284
+ def physical_agent_fqn(agent: dict[str, Any]) -> str:
285
+ return fqn(str(agent.get("physical_fqn") or ""), f"physical Agent for {agent.get('name')}")
@@ -0,0 +1,181 @@
1
+ from __future__ import annotations
2
+
3
+ import fnmatch
4
+ from dataclasses import dataclass
5
+ from pathlib import Path
6
+
7
+ from .config import Config
8
+ from .dbt_runner import CommandRunner
9
+ from .manifest import SkillDeclaration, select_agents, skill_declarations, stage_path_parts
10
+
11
+
12
+ @dataclass(frozen=True)
13
+ class SkillUpload:
14
+ stage_fqn: str
15
+ stage_path: str
16
+ local_dir: Path
17
+ skill_names: tuple[str, ...]
18
+ agent_names: tuple[str, ...]
19
+
20
+
21
+ def _stage_parts(stage_path: str) -> tuple[str, str]:
22
+ return stage_path_parts(stage_path)
23
+
24
+
25
+ def stage_database(stage_path: str) -> str:
26
+ stage_fqn, _ = _stage_parts(stage_path)
27
+ return stage_fqn.split(".", 1)[0]
28
+
29
+
30
+ def _ignored_private_skill(project_dir: Path, local_dir: Path) -> bool:
31
+ try:
32
+ relative = local_dir.resolve().relative_to(project_dir.resolve()).as_posix()
33
+ except ValueError:
34
+ return False
35
+ if not relative.startswith("models/agents/") or "/skills/" not in relative:
36
+ return True
37
+ ignore_file = project_dir / ".dbtignore"
38
+ if not ignore_file.is_file():
39
+ return False
40
+ candidate = f"{relative}/SKILL.md"
41
+ patterns = [
42
+ line.strip()
43
+ for line in ignore_file.read_text(encoding="utf-8").splitlines()
44
+ if line.strip() and not line.lstrip().startswith("#") and not line.startswith("!")
45
+ ]
46
+ return any(fnmatch.fnmatch(candidate, pattern) for pattern in patterns)
47
+
48
+
49
+ def build_upload_plan(
50
+ manifest: dict, project_dir: str | Path, agent_names: list[str] | None = None
51
+ ) -> list[SkillUpload]:
52
+ project_path = Path(project_dir).resolve()
53
+ for agent in select_agents(manifest, agent_names):
54
+ names: set[str] = set()
55
+ configured = agent["meta"].get("skills") or []
56
+ spec_skills = agent["meta"].get("compiled_spec", {}).get("skills") or []
57
+ for skill in configured or spec_skills:
58
+ skill_name = skill.get("name")
59
+ if skill_name in names:
60
+ raise ValueError(
61
+ f"Agent {agent['name']!r} declares duplicate skill name {skill_name!r}"
62
+ )
63
+ if skill_name:
64
+ names.add(skill_name)
65
+ declarations = skill_declarations(manifest, project_path, agent_names)
66
+ stage_sources: dict[str, tuple[str, Path]] = {}
67
+ grouped: dict[str, list[SkillDeclaration]] = {}
68
+
69
+ for declaration in declarations:
70
+ _stage_parts(declaration.stage_path)
71
+ local_dir = declaration.local_dir.resolve()
72
+ skill_file = local_dir / "SKILL.md"
73
+ if not skill_file.is_file():
74
+ raise FileNotFoundError(
75
+ f"Declared skill {declaration.skill_name!r} for Agent "
76
+ f"{declaration.agent_name!r} is missing {skill_file}"
77
+ )
78
+ if not _ignored_private_skill(project_path, local_dir):
79
+ raise ValueError(
80
+ f"Private skill directory {local_dir} is not protected by "
81
+ f"{project_path / '.dbtignore'}"
82
+ )
83
+ collision_key = declaration.stage_path.casefold()
84
+ prior = stage_sources.get(collision_key)
85
+ if prior is not None and prior[1] != local_dir:
86
+ raise ValueError(
87
+ f"Skill stage path collision for {declaration.stage_path}: {prior[1]} and {local_dir}"
88
+ )
89
+ if prior is None:
90
+ stage_sources[collision_key] = (declaration.stage_path, local_dir)
91
+ grouped.setdefault(collision_key, []).append(declaration)
92
+
93
+ plan: list[SkillUpload] = []
94
+ for collision_key, values in grouped.items():
95
+ stage_path, local_dir = stage_sources[collision_key]
96
+ stage_fqn, _ = _stage_parts(stage_path)
97
+ plan.append(
98
+ SkillUpload(
99
+ stage_fqn=stage_fqn,
100
+ stage_path=stage_path,
101
+ local_dir=local_dir,
102
+ skill_names=tuple(sorted({item.skill_name for item in values})),
103
+ agent_names=tuple(sorted({item.agent_name for item in values})),
104
+ )
105
+ )
106
+ return sorted(plan, key=lambda item: item.stage_path.casefold())
107
+
108
+
109
+ def assert_apply_safety(
110
+ config: Config, allowed_targets: list[str], allowed_databases: list[str]
111
+ ) -> None:
112
+ if not config.target:
113
+ raise ValueError("Apply requires an explicit --target or DBT_TARGET")
114
+ if config.target not in allowed_targets:
115
+ raise ValueError(
116
+ f"Refusing apply for target {config.target!r}; allowed targets: "
117
+ f"{', '.join(allowed_targets) or 'none'}"
118
+ )
119
+ if not config.database:
120
+ raise ValueError("Apply requires an explicit --database or SNOWFLAKE_DATABASE")
121
+ allowed = {name.upper() for name in allowed_databases}
122
+ if config.database.upper() not in allowed:
123
+ raise ValueError(
124
+ f"Refusing apply for database {config.database!r}; allowed databases: "
125
+ f"{', '.join(allowed_databases) or 'none'}"
126
+ )
127
+ if not config.connection_explicit or not config.connection:
128
+ raise ValueError("Apply requires an explicitly supplied --connection")
129
+
130
+
131
+ def upload_skills(
132
+ plan: list[SkillUpload], config: Config, runner: CommandRunner | None = None
133
+ ) -> None:
134
+ command_runner = runner or CommandRunner()
135
+ validated: set[str] = set()
136
+ for upload in plan:
137
+ stage_key = upload.stage_fqn.casefold()
138
+ if stage_key in validated:
139
+ continue
140
+ result = command_runner.run(
141
+ [
142
+ config.snow_executable,
143
+ "sql",
144
+ "--connection",
145
+ str(config.connection),
146
+ "--query",
147
+ f"DESCRIBE STAGE {upload.stage_fqn}",
148
+ ],
149
+ cwd=config.project_dir,
150
+ )
151
+ if result.returncode != 0:
152
+ detail = result.stderr.strip() or result.stdout.strip() or "stage is missing"
153
+ raise RuntimeError(
154
+ f"Skill stage {upload.stage_fqn} is missing or inaccessible; "
155
+ "provision it through the environment infrastructure owner before "
156
+ f"uploading skills: {detail}"
157
+ )
158
+ validated.add(stage_key)
159
+
160
+ for upload in plan:
161
+ result = command_runner.run(
162
+ [
163
+ config.snow_executable,
164
+ "stage",
165
+ "copy",
166
+ "--connection",
167
+ str(config.connection),
168
+ str(upload.local_dir / "*"),
169
+ f"{upload.stage_path}/",
170
+ "--overwrite",
171
+ "--recursive",
172
+ "--parallel",
173
+ "4",
174
+ ],
175
+ cwd=config.project_dir,
176
+ )
177
+ if result.returncode != 0:
178
+ names = ", ".join(upload.skill_names)
179
+ raise RuntimeError(
180
+ result.stderr.strip() or result.stdout.strip() or f"skill upload failed: {names}"
181
+ )
@@ -0,0 +1,11 @@
1
+ from __future__ import annotations
2
+
3
+ import subprocess
4
+
5
+ from .dbt_runner import CommandRunner
6
+
7
+
8
+ def connection_test(
9
+ executable: str, connection: str, runner: CommandRunner
10
+ ) -> subprocess.CompletedProcess[str]:
11
+ return runner.run([executable, "connection", "test", "--connection", connection])
@@ -0,0 +1,7 @@
1
+ version: 2
2
+
3
+ models:
4
+ - name: orders_assistant
5
+ description: Full-body Cortex Agent model for governed order analytics.
6
+ config:
7
+ access: protected
@@ -0,0 +1,54 @@
1
+ version: 2
2
+
3
+ models:
4
+ - name: orders_assistant_core
5
+ description: Minimal evaluation dataset for Orders Assistant.
6
+ config:
7
+ materialized: table
8
+ schema: eval
9
+ meta:
10
+ cortex_eval:
11
+ name: core
12
+ agent: orders_assistant
13
+ description: Core routing, correctness, and boundary checks.
14
+ metrics:
15
+ - answer_correctness
16
+ - logical_consistency
17
+ - tool_selection_accuracy
18
+ - tool_execution_accuracy
19
+ thresholds:
20
+ answer_correctness: 0.60
21
+ logical_consistency: 0.60
22
+ tool_selection_accuracy: 0.80
23
+ tool_execution_accuracy: 0.70
24
+ questions:
25
+ - id: total_revenue
26
+ test_type: in_scope
27
+ expected_tools: [OrdersAnalytics]
28
+ ground_truth_ref: total_revenue
29
+ - id: out_of_scope_competitor_inventory
30
+ test_type: out_of_scope
31
+ expected_tools: []
32
+ ground_truth_ref: out_of_scope_competitor_inventory
33
+ - id: negative_future_revenue
34
+ test_type: negative
35
+ expected_tools: []
36
+ ground_truth_ref: negative_future_revenue
37
+ data_tests:
38
+ - dbt_cortex_agent.cortex_eval_question_coverage:
39
+ arguments:
40
+ min_out_of_scope: 1
41
+ min_negative: 1
42
+ expected_tools: [OrdersAnalytics]
43
+ expected_refs:
44
+ - total_revenue
45
+ - out_of_scope_competitor_inventory
46
+ - negative_future_revenue
47
+ columns:
48
+ - name: input_query
49
+ data_tests:
50
+ - not_null
51
+ - unique
52
+ - name: output
53
+ data_tests:
54
+ - not_null
@@ -0,0 +1,50 @@
1
+ {{ config(materialized='table', schema='eval') }}
2
+
3
+ WITH total_revenue AS (
4
+ SELECT
5
+ 'What was total order revenue?' AS input_query,
6
+ 'Total order revenue was $' || TO_VARCHAR(ROUND(COALESCE(SUM(revenue), 0), 2), 'FM9999999990.00') || '.' AS ground_truth_output,
7
+ ARRAY_CONSTRUCT(
8
+ OBJECT_CONSTRUCT(
9
+ 'tool_name', 'OrdersAnalytics',
10
+ 'tool_input', 'Calculate total order revenue.',
11
+ 'tool_output', 'Semantic view result with total_revenue.'
12
+ )
13
+ ) AS ground_truth_invocations,
14
+ OBJECT_CONSTRUCT(
15
+ 'category', 'orders',
16
+ 'test_type', 'in_scope',
17
+ 'ground_truth_ref', 'total_revenue'
18
+ ) AS custom_criteria
19
+ FROM {{ ref('orders') }}
20
+ ),
21
+
22
+ out_of_scope_competitor_inventory AS (
23
+ SELECT
24
+ 'How many units does our largest competitor have in inventory?' AS input_query,
25
+ 'The Agent should decline because competitor inventory is outside the governed order data.' AS ground_truth_output,
26
+ ARRAY_CONSTRUCT() AS ground_truth_invocations,
27
+ OBJECT_CONSTRUCT(
28
+ 'category', 'boundary',
29
+ 'test_type', 'out_of_scope',
30
+ 'ground_truth_ref', 'out_of_scope_competitor_inventory'
31
+ ) AS custom_criteria
32
+ ),
33
+
34
+ negative_future_revenue AS (
35
+ SELECT
36
+ 'What will exact order revenue be next year?' AS input_query,
37
+ 'The Agent should explain that the available order history cannot determine exact future revenue.' AS ground_truth_output,
38
+ ARRAY_CONSTRUCT() AS ground_truth_invocations,
39
+ OBJECT_CONSTRUCT(
40
+ 'category', 'boundary',
41
+ 'test_type', 'negative',
42
+ 'ground_truth_ref', 'negative_future_revenue'
43
+ ) AS custom_criteria
44
+ )
45
+
46
+ {{ dbt_cortex_agent.cortex_eval__assemble([
47
+ 'total_revenue',
48
+ 'out_of_scope_competitor_inventory',
49
+ 'negative_future_revenue'
50
+ ]) }}
@@ -0,0 +1,54 @@
1
+ {% do ref('sem_orders') %}
2
+ {{
3
+ config(
4
+ materialized='cortex_agent',
5
+ database=target.database,
6
+ schema='AGENTS',
7
+ alias='ORDERS_ASSISTANT_SANDBOX',
8
+ meta={
9
+ 'agent_display_name': 'Orders Assistant',
10
+ 'agent_comment': 'Governed order analytics assistant',
11
+ 'deploy_alias': 'latest',
12
+ 'cortex_agent': {
13
+ 'enabled': true,
14
+ 'access': {
15
+ 'usage_roles': [env_var('CORTEX_AGENT_RUNTIME_ROLE', 'ORDERS_AGENT_USER')],
16
+ 'monitor_roles': [env_var('CORTEX_AGENT_MONITOR_ROLE', 'ORDERS_AGENT_MONITOR')]
17
+ },
18
+ 'evaluation': {'native_tools': ['OrdersAnalytics']}
19
+ }
20
+ }
21
+ )
22
+ }}
23
+
24
+ models:
25
+ orchestration: claude-sonnet-4-6
26
+
27
+ orchestration:
28
+ budget:
29
+ seconds: 60
30
+ tokens: 16000
31
+
32
+ instructions:
33
+ orchestration: Use OrdersAnalytics for governed order questions. Do not invent unavailable facts.
34
+ response: Answer concisely and state the requested time scope and units.
35
+ sample_questions:
36
+ - question: What was total order revenue?
37
+ - question: Which region generated the most revenue?
38
+
39
+ tools:
40
+ - tool_spec:
41
+ type: cortex_analyst_text_to_sql
42
+ name: OrdersAnalytics
43
+ description: |
44
+ Analyzes governed order revenue, count, average value, dates, and regions.
45
+ Use for aggregate order questions and comparisons. Not for competitor data,
46
+ inventory, support tickets, or forecasts.
47
+
48
+ tool_resources:
49
+ OrdersAnalytics:
50
+ semantic_view: "{{ target.database }}.SEMANTIC.SEM_ORDERS"
51
+ execution_environment:
52
+ type: warehouse
53
+ warehouse: "{{ target.warehouse }}"
54
+ query_timeout: 60
@@ -0,0 +1,40 @@
1
+ {{ config(materialized='semantic_view') }}
2
+
3
+ TABLES (
4
+ orders AS {{ ref('orders') }}
5
+ PRIMARY KEY (order_id)
6
+ WITH SYNONYMS ('sales orders')
7
+ COMMENT = 'Completed customer orders'
8
+ )
9
+
10
+ FACTS (
11
+ orders.revenue AS revenue
12
+ COMMENT = 'Revenue recorded for the order'
13
+ )
14
+
15
+ DIMENSIONS (
16
+ orders.order_date AS order_date
17
+ WITH SYNONYMS ('sale date')
18
+ COMMENT = 'Date the order was completed',
19
+ orders.region AS region
20
+ WITH SYNONYMS ('sales region')
21
+ COMMENT = 'Region credited with the order'
22
+ )
23
+
24
+ METRICS (
25
+ orders.total_revenue AS SUM(orders.revenue)
26
+ WITH SYNONYMS ('sales', 'revenue')
27
+ COMMENT = 'Total order revenue',
28
+ orders.total_orders AS COUNT(orders.order_id)
29
+ WITH SYNONYMS ('order count')
30
+ COMMENT = 'Number of completed orders',
31
+ orders.average_order_value AS AVG(orders.revenue)
32
+ WITH SYNONYMS ('average sale')
33
+ COMMENT = 'Average revenue per order'
34
+ )
35
+
36
+ COMMENT = 'Starter semantic view for governed order analysis'
37
+
38
+ AI_SQL_GENERATION 'Use only the declared dimensions and metrics. Use order_date for time filters and region for geographic comparisons.'
39
+
40
+ AI_QUESTION_CATEGORIZATION 'Accept questions about order revenue, count, average value, dates, and regions. Reject unrelated operational or competitor questions.'