planwise 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- planwise-0.1.0/.claude/settings.local.json +7 -0
- planwise-0.1.0/.gitignore +9 -0
- planwise-0.1.0/Dockerfile +20 -0
- planwise-0.1.0/LICENSE +21 -0
- planwise-0.1.0/PKG-INFO +100 -0
- planwise-0.1.0/README.md +55 -0
- planwise-0.1.0/apiproxy.cdsys.crt +38 -0
- planwise-0.1.0/cdsys-ca-bundle.pem +59 -0
- planwise-0.1.0/cdsys-ca.crt +21 -0
- planwise-0.1.0/data/golden_plans.json +230 -0
- planwise-0.1.0/data/sample_catalog.json +496 -0
- planwise-0.1.0/data/sample_queries.json +25 -0
- planwise-0.1.0/docs/HLD.md +118 -0
- planwise-0.1.0/docs/LLD.md +271 -0
- planwise-0.1.0/docs/VERIFICATION.md +93 -0
- planwise-0.1.0/docs/architecture-diagram.html +288 -0
- planwise-0.1.0/docs/deployment-strategies.md +208 -0
- planwise-0.1.0/eval/__init__.py +0 -0
- planwise-0.1.0/eval/harness.py +275 -0
- planwise-0.1.0/eval/perturb.py +78 -0
- planwise-0.1.0/eval/plan_eval.py +201 -0
- planwise-0.1.0/eval/viewer.py +90 -0
- planwise-0.1.0/examples/tier1_http.py +35 -0
- planwise-0.1.0/examples/tier2_wrapper.py +62 -0
- planwise-0.1.0/examples/tier3_mcp_config.json +9 -0
- planwise-0.1.0/planwise/__init__.py +8 -0
- planwise-0.1.0/planwise/cli.py +91 -0
- planwise-0.1.0/planwise/config.py +126 -0
- planwise-0.1.0/planwise/engine/__init__.py +0 -0
- planwise-0.1.0/planwise/engine/adapters.py +111 -0
- planwise-0.1.0/planwise/engine/assembler.py +133 -0
- planwise-0.1.0/planwise/engine/binder.py +127 -0
- planwise-0.1.0/planwise/engine/embeddings.py +88 -0
- planwise-0.1.0/planwise/engine/enrich.py +86 -0
- planwise-0.1.0/planwise/engine/entities.py +84 -0
- planwise-0.1.0/planwise/engine/graphs.py +124 -0
- planwise-0.1.0/planwise/engine/index.py +177 -0
- planwise-0.1.0/planwise/engine/intake.py +120 -0
- planwise-0.1.0/planwise/engine/llm.py +116 -0
- planwise-0.1.0/planwise/engine/models.py +183 -0
- planwise-0.1.0/planwise/engine/plan_validator.py +153 -0
- planwise-0.1.0/planwise/engine/planlog.py +30 -0
- planwise-0.1.0/planwise/engine/planner.py +360 -0
- planwise-0.1.0/planwise/engine/router.py +287 -0
- planwise-0.1.0/planwise/engine/scorer.py +138 -0
- planwise-0.1.0/planwise/mcp_server.py +162 -0
- planwise-0.1.0/planwise/middleware/__init__.py +6 -0
- planwise-0.1.0/planwise/middleware/interceptors.py +211 -0
- planwise-0.1.0/planwise/middleware/widening.py +69 -0
- planwise-0.1.0/planwise/middleware/wrapper.py +188 -0
- planwise-0.1.0/planwise/server.py +141 -0
- planwise-0.1.0/pyproject.toml +73 -0
- planwise-0.1.0/tests/__init__.py +0 -0
- planwise-0.1.0/tests/conftest.py +65 -0
- planwise-0.1.0/tests/test_binding_regressions.py +82 -0
- planwise-0.1.0/tests/test_llm.py +80 -0
- planwise-0.1.0/tests/test_mcp.py +50 -0
- planwise-0.1.0/tests/test_middleware.py +102 -0
- planwise-0.1.0/tests/test_perturb.py +61 -0
- planwise-0.1.0/tests/test_planner.py +278 -0
- planwise-0.1.0/tests/test_router.py +135 -0
- planwise-0.1.0/tests/test_server.py +73 -0
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
# PlanWise Router — Tier 1 HTTP gateway image.
|
|
2
|
+
# Ships the semantic + server extras; the LLM planner is optional at runtime
|
|
3
|
+
# (set ANTHROPIC_API_KEY to enable it, otherwise the rules planner is used).
|
|
4
|
+
FROM python:3.11-slim
|
|
5
|
+
|
|
6
|
+
WORKDIR /app
|
|
7
|
+
|
|
8
|
+
# Install deps first for layer caching.
|
|
9
|
+
COPY pyproject.toml README.md ./
|
|
10
|
+
COPY planwise ./planwise
|
|
11
|
+
RUN pip install --no-cache-dir ".[server,embeddings]"
|
|
12
|
+
|
|
13
|
+
COPY data ./data
|
|
14
|
+
|
|
15
|
+
ENV PLANWISE_CATALOG=data/sample_catalog.json
|
|
16
|
+
EXPOSE 8000
|
|
17
|
+
|
|
18
|
+
# Boot the gateway. Override the catalog by mounting a file and setting
|
|
19
|
+
# PLANWISE_CATALOG, or POST your own to /catalog at runtime.
|
|
20
|
+
CMD ["planwise", "serve", "--host", "0.0.0.0", "--port", "8000"]
|
planwise-0.1.0/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Crest Data AI
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
planwise-0.1.0/PKG-INFO
ADDED
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: planwise
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: PlanWise Router — intelligent tool routing layer for large tool catalogs
|
|
5
|
+
Project-URL: Repository, https://gitlab.com/mygroup4322601/planwise
|
|
6
|
+
Author-email: Crest Data AI <rikin.solanki@crestdata.ai>
|
|
7
|
+
License: MIT
|
|
8
|
+
License-File: LICENSE
|
|
9
|
+
Classifier: Development Status :: 4 - Beta
|
|
10
|
+
Classifier: Intended Audience :: Developers
|
|
11
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
12
|
+
Classifier: Programming Language :: Python :: 3
|
|
13
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
14
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
15
|
+
Classifier: Topic :: Software Development :: Libraries
|
|
16
|
+
Requires-Python: >=3.11
|
|
17
|
+
Requires-Dist: jsonschema>=4.21
|
|
18
|
+
Requires-Dist: numpy>=1.26
|
|
19
|
+
Requires-Dist: pydantic-settings>=2.2
|
|
20
|
+
Requires-Dist: pydantic>=2.7
|
|
21
|
+
Requires-Dist: rank-bm25>=0.2.2
|
|
22
|
+
Provides-Extra: dev
|
|
23
|
+
Requires-Dist: fastapi>=0.110; extra == 'dev'
|
|
24
|
+
Requires-Dist: httpx; extra == 'dev'
|
|
25
|
+
Requires-Dist: httpx>=0.27; extra == 'dev'
|
|
26
|
+
Requires-Dist: mcp>=1.2; extra == 'dev'
|
|
27
|
+
Requires-Dist: pytest>=8.0; extra == 'dev'
|
|
28
|
+
Requires-Dist: ruff>=0.4; extra == 'dev'
|
|
29
|
+
Requires-Dist: scikit-learn>=1.4; extra == 'dev'
|
|
30
|
+
Requires-Dist: sentence-transformers>=3.0; extra == 'dev'
|
|
31
|
+
Requires-Dist: uvicorn>=0.29; extra == 'dev'
|
|
32
|
+
Provides-Extra: embeddings
|
|
33
|
+
Requires-Dist: scikit-learn>=1.4; extra == 'embeddings'
|
|
34
|
+
Requires-Dist: sentence-transformers>=3.0; extra == 'embeddings'
|
|
35
|
+
Provides-Extra: llm
|
|
36
|
+
Requires-Dist: httpx>=0.27; extra == 'llm'
|
|
37
|
+
Provides-Extra: mcp
|
|
38
|
+
Requires-Dist: mcp>=1.2; extra == 'mcp'
|
|
39
|
+
Provides-Extra: server
|
|
40
|
+
Requires-Dist: fastapi>=0.110; extra == 'server'
|
|
41
|
+
Requires-Dist: uvicorn>=0.29; extra == 'server'
|
|
42
|
+
Provides-Extra: viewer
|
|
43
|
+
Requires-Dist: streamlit>=1.30; extra == 'viewer'
|
|
44
|
+
Description-Content-Type: text/markdown
|
|
45
|
+
|
|
46
|
+
# PlanWise Router
|
|
47
|
+
|
|
48
|
+
An intelligent tool-routing layer for large tool catalogs (60+ tools). Instead of showing an AI agent every tool schema — which wastes tokens and causes wrong picks between look-alike tools — PlanWise reads the user's query, writes a step-by-step plan, and hands the agent only the 3–6 tools it actually needs, in the right order.
|
|
49
|
+
|
|
50
|
+
Built for the Crest Data AI Innovation Challenge (Problem 02).
|
|
51
|
+
|
|
52
|
+
## How it works (short version)
|
|
53
|
+
|
|
54
|
+
1. **Offline:** every tool passes quality gates, gets ~8 LLM-written example user questions, and is indexed (embeddings + keywords). Twin tools are grouped; version families (v1/v2) are linked.
|
|
55
|
+
2. **Runtime:** one cheap LLM call breaks the query into abstract steps (it never sees tool names, so it can't invent one). Each step finds its best tool with a 4-signal score: meaning match + keyword match + input fit + **output fit** (what the tool returns vs what the step needs — this is how same-input twins get told apart).
|
|
56
|
+
3. **Hand-off:** the agent gets the few chosen tool schemas plus a plan hint like `Suggested plan: 1) get_revenue_report 2) generate_chart 3) send_email`.
|
|
57
|
+
|
|
58
|
+
Full design: [docs/HLD.md](docs/HLD.md) · [docs/LLD.md](docs/LLD.md) · [docs/architecture-diagram.html](docs/architecture-diagram.html)
|
|
59
|
+
|
|
60
|
+
## Quick start
|
|
61
|
+
|
|
62
|
+
```bash
|
|
63
|
+
pip install -e ".[dev]" # or: pip install -e . for the minimal core
|
|
64
|
+
|
|
65
|
+
export $(grep -v '^#' .env | xargs) # if inside venv
|
|
66
|
+
|
|
67
|
+
# Route one query against the sample catalog
|
|
68
|
+
planwise route "Pull last quarter's revenue and email the chart to finance@corp.com"
|
|
69
|
+
|
|
70
|
+
# Run the evaluation harness (with the naive baseline to beat)
|
|
71
|
+
planwise eval --baseline
|
|
72
|
+
```
|
|
73
|
+
|
|
74
|
+
Works fully offline out of the box: with no gateway configured it uses a rule-based planner and lexical embeddings. For the real pipeline (LLM planner + semantic embeddings), point it at the Crest AWS Bedrock gateway (OpenAI-compatible):
|
|
75
|
+
|
|
76
|
+
```bash
|
|
77
|
+
pip install -e ".[llm,embeddings]"
|
|
78
|
+
export PLANWISE_LLM_API_KEY=sk-... # your gateway key
|
|
79
|
+
export PLANWISE_LLM_CERT=/path/to/apiproxy.cdsys.crt # internal-CA cert from IT
|
|
80
|
+
# must be on the Crest network / VPN
|
|
81
|
+
planwise eval --baseline
|
|
82
|
+
```
|
|
83
|
+
|
|
84
|
+
All LLM traffic goes through `planwise/engine/llm.py` (one place). The key and cert path come from the environment — never hardcoded or committed. Default models: `Bedrock-ant-haiku-4-5-20251001-v1-0` for the planner/enrichment.
|
|
85
|
+
|
|
86
|
+
## Layout
|
|
87
|
+
|
|
88
|
+
- `planwise/engine/` — the routing engine (pure library: intake, index, planner, scorer, binder, assembler)
|
|
89
|
+
- `planwise/middleware/` — self-healing execution wrapper (Tier 2)
|
|
90
|
+
- `planwise/server.py` — HTTP gateway (Tier 1), `planwise/mcp_server.py` — MCP surface (Tier 3)
|
|
91
|
+
- `eval/` — evaluation harness, stress suites, baselines
|
|
92
|
+
- `data/` — sample catalog + ground-truth queries (replaced by the organizers' files at hackathon start)
|
|
93
|
+
|
|
94
|
+
## Tests
|
|
95
|
+
|
|
96
|
+
```bash
|
|
97
|
+
python -m pytest tests/
|
|
98
|
+
```
|
|
99
|
+
|
|
100
|
+
Tests run deterministic and offline (hashing embeddings, rules planner) — no downloads, no API keys.
|
planwise-0.1.0/README.md
ADDED
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
# PlanWise Router
|
|
2
|
+
|
|
3
|
+
An intelligent tool-routing layer for large tool catalogs (60+ tools). Instead of showing an AI agent every tool schema — which wastes tokens and causes wrong picks between look-alike tools — PlanWise reads the user's query, writes a step-by-step plan, and hands the agent only the 3–6 tools it actually needs, in the right order.
|
|
4
|
+
|
|
5
|
+
Built for the Crest Data AI Innovation Challenge (Problem 02).
|
|
6
|
+
|
|
7
|
+
## How it works (short version)
|
|
8
|
+
|
|
9
|
+
1. **Offline:** every tool passes quality gates, gets ~8 LLM-written example user questions, and is indexed (embeddings + keywords). Twin tools are grouped; version families (v1/v2) are linked.
|
|
10
|
+
2. **Runtime:** one cheap LLM call breaks the query into abstract steps (it never sees tool names, so it can't invent one). Each step finds its best tool with a 4-signal score: meaning match + keyword match + input fit + **output fit** (what the tool returns vs what the step needs — this is how same-input twins get told apart).
|
|
11
|
+
3. **Hand-off:** the agent gets the few chosen tool schemas plus a plan hint like `Suggested plan: 1) get_revenue_report 2) generate_chart 3) send_email`.
|
|
12
|
+
|
|
13
|
+
Full design: [docs/HLD.md](docs/HLD.md) · [docs/LLD.md](docs/LLD.md) · [docs/architecture-diagram.html](docs/architecture-diagram.html)
|
|
14
|
+
|
|
15
|
+
## Quick start
|
|
16
|
+
|
|
17
|
+
```bash
|
|
18
|
+
pip install -e ".[dev]" # or: pip install -e . for the minimal core
|
|
19
|
+
|
|
20
|
+
export $(grep -v '^#' .env | xargs) # if inside venv
|
|
21
|
+
|
|
22
|
+
# Route one query against the sample catalog
|
|
23
|
+
planwise route "Pull last quarter's revenue and email the chart to finance@corp.com"
|
|
24
|
+
|
|
25
|
+
# Run the evaluation harness (with the naive baseline to beat)
|
|
26
|
+
planwise eval --baseline
|
|
27
|
+
```
|
|
28
|
+
|
|
29
|
+
Works fully offline out of the box: with no gateway configured it uses a rule-based planner and lexical embeddings. For the real pipeline (LLM planner + semantic embeddings), point it at the Crest AWS Bedrock gateway (OpenAI-compatible):
|
|
30
|
+
|
|
31
|
+
```bash
|
|
32
|
+
pip install -e ".[llm,embeddings]"
|
|
33
|
+
export PLANWISE_LLM_API_KEY=sk-... # your gateway key
|
|
34
|
+
export PLANWISE_LLM_CERT=/path/to/apiproxy.cdsys.crt # internal-CA cert from IT
|
|
35
|
+
# must be on the Crest network / VPN
|
|
36
|
+
planwise eval --baseline
|
|
37
|
+
```
|
|
38
|
+
|
|
39
|
+
All LLM traffic goes through `planwise/engine/llm.py` (one place). The key and cert path come from the environment — never hardcoded or committed. Default models: `Bedrock-ant-haiku-4-5-20251001-v1-0` for the planner/enrichment.
|
|
40
|
+
|
|
41
|
+
## Layout
|
|
42
|
+
|
|
43
|
+
- `planwise/engine/` — the routing engine (pure library: intake, index, planner, scorer, binder, assembler)
|
|
44
|
+
- `planwise/middleware/` — self-healing execution wrapper (Tier 2)
|
|
45
|
+
- `planwise/server.py` — HTTP gateway (Tier 1), `planwise/mcp_server.py` — MCP surface (Tier 3)
|
|
46
|
+
- `eval/` — evaluation harness, stress suites, baselines
|
|
47
|
+
- `data/` — sample catalog + ground-truth queries (replaced by the organizers' files at hackathon start)
|
|
48
|
+
|
|
49
|
+
## Tests
|
|
50
|
+
|
|
51
|
+
```bash
|
|
52
|
+
python -m pytest tests/
|
|
53
|
+
```
|
|
54
|
+
|
|
55
|
+
Tests run deterministic and offline (hashing embeddings, rules planner) — no downloads, no API keys.
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
-----BEGIN CERTIFICATE-----
|
|
2
|
+
MIIGkTCCBXmgAwIBAgITVQAAAvIbmEuSWZ/3sAAAAAAC8jANBgkqhkiG9w0BAQsF
|
|
3
|
+
ADBBMRUwEwYKCZImiZPyLGQBGRYFTE9DQUwxFTATBgoJkiaJk/IsZAEZFgVDRFNZ
|
|
4
|
+
UzERMA8GA1UEAxMIQ0RTWVMtQ0EwHhcNMjYwNTE4MTIzNDI5WhcNMjcwNTE4MTI0
|
|
5
|
+
NDI5WjCBhzELMAkGA1UEBhMCSU4xEDAOBgNVBAgTB0d1amFyYXQxEjAQBgNVBAcT
|
|
6
|
+
CUFobWVkYWJhZDESMBAGA1UEChMJQ3Jlc3REYXRhMR8wHQYDVQQLExZJbmZvcm1h
|
|
7
|
+
dGlvbiBUZWNobm9sb2d5MR0wGwYDVQQDExRhcGlwcm94eS5jZHN5cy5sb2NhbDCC
|
|
8
|
+
AiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBALJaNBpJuncXzdv/VFagnmNS
|
|
9
|
+
Sh/ArM+ZrvScI36w8jusjzDrJzHLU1VqpHUthRjmkpD3LG1xBAL4RQnc6dlPuyeg
|
|
10
|
+
e1B4TimyFoIk6ZkrPjBvDECNigFbZdZDyRbc/4q3yGPSwFqC4mZfVYuy4rbNFeBz
|
|
11
|
+
Qhbg7rYAppYxN8rV+F8GXoYSLL4hKQiaGdhi8dU3V1rkOXFjsSKlysCa8TBI8Ngn
|
|
12
|
+
bopAvD2bu3QMF6O5OeQlGiY/f4S9RGenPXPRUAc/xPYHD4LKdVs0kYPNhB6wYNoz
|
|
13
|
+
9pEM3uxwsBrjyLhplhQQTY/3stHhzq5TKYpe+DXfIhZqeczK/iHOtFbb4kteKWg4
|
|
14
|
+
hePdE68UqW1abqWPpt1Uqe2QXBfJ/HlRkuyJ074uynXWr/D6+6U2MhjxeOvA7qKr
|
|
15
|
+
AyeIKtEp/ZOnjXyj9kAsqkcYH8DY94U6NPu1wjvVFTc9H3p95+7Nldcy0rWnyFTp
|
|
16
|
+
gdNPwgj7FhTynYoPlaznsuyCMwTI4rXIvNA4eCFYFuPUKR2xpoWEh3itAcY0Y4Ar
|
|
17
|
+
GEBxIAHXNvXMIakLcKsGd6YShOMBWnfvVuhobyFEdPKfXIU8JS3MedFvsKgWkPLT
|
|
18
|
+
/JnkoDcwxCxIgBtMx1FaRTpIOzvhjjjExKUuyjYaWh3ChJVLaZHs3QUCuAm9jDol
|
|
19
|
+
3iy2IvQxuOIVuiYZNnaLAgMBAAGjggI5MIICNTAOBgNVHQ8BAf8EBAMCBaAwEwYD
|
|
20
|
+
VR0lBAwwCgYIKwYBBQUHAwEwJQYDVR0RBB4wHIIUYXBpcHJveHkuY2RzeXMubG9j
|
|
21
|
+
YWyHBAoyBr8wHQYDVR0OBBYEFCHwfcUcIpT12WQbt/Dalved8TIoMB8GA1UdIwQY
|
|
22
|
+
MBaAFHH/eA+3icaHbj3eBS2KpGmte+MhMIHGBgNVHR8Egb4wgbswgbiggbWggbKG
|
|
23
|
+
ga9sZGFwOi8vL0NOPUNEU1lTLUNBLENOPUNEU0RDMDEsQ049Q0RQLENOPVB1Ymxp
|
|
24
|
+
YyUyMEtleSUyMFNlcnZpY2VzLENOPVNlcnZpY2VzLENOPUNvbmZpZ3VyYXRpb24s
|
|
25
|
+
REM9Q0RTWVMsREM9TE9DQUw/Y2VydGlmaWNhdGVSZXZvY2F0aW9uTGlzdD9iYXNl
|
|
26
|
+
P29iamVjdENsYXNzPWNSTERpc3RyaWJ1dGlvblBvaW50MIG6BggrBgEFBQcBAQSB
|
|
27
|
+
rTCBqjCBpwYIKwYBBQUHMAKGgZpsZGFwOi8vL0NOPUNEU1lTLUNBLENOPUFJQSxD
|
|
28
|
+
Tj1QdWJsaWMlMjBLZXklMjBTZXJ2aWNlcyxDTj1TZXJ2aWNlcyxDTj1Db25maWd1
|
|
29
|
+
cmF0aW9uLERDPUNEU1lTLERDPUxPQ0FMP2NBQ2VydGlmaWNhdGU/YmFzZT9vYmpl
|
|
30
|
+
Y3RDbGFzcz1jZXJ0aWZpY2F0aW9uQXV0aG9yaXR5MCEGCSsGAQQBgjcUAgQUHhIA
|
|
31
|
+
VwBlAGIAUwBlAHIAdgBlAHIwDQYJKoZIhvcNAQELBQADggEBAFqh5t3037Tj8e/2
|
|
32
|
+
+5bwi/o3LQbNS4YXtmXYLu5rMXCJMaXthp9ypjMQTB6OP3m2R+zQx4rgBYevH0Ju
|
|
33
|
+
sjnw09w/571EsMY+Oh8Ehd3qwzfx+QxJ8loIsjaDE7xBnuzKxjXLY+Ritkt2yDmb
|
|
34
|
+
fgTCHMxIXJu4riFsMNuwwd7j6HOBP4pmymmhmFlO/kRtd7gn87eZlEfNSBNNq5pa
|
|
35
|
+
WLVNW0Jg1jrLAQwQfPe9mX5ID9xC9n46vSnbDukXR380zc9N55UrkpIHeU+VMQZh
|
|
36
|
+
AknBywOdKV1gFnwj2ItZT4wtLX42mXkCtH1KuJ0+LC2yLrswd0jT/Bpnimg0lyjh
|
|
37
|
+
AuEDBcQ=
|
|
38
|
+
-----END CERTIFICATE-----
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
-----BEGIN CERTIFICATE-----
|
|
2
|
+
MIIDdzCCAl+gAwIBAgIQOxQW8dLPJLdAoIx8ZgQzVzANBgkqhkiG9w0BAQsFADBB
|
|
3
|
+
MRUwEwYKCZImiZPyLGQBGRYFTE9DQUwxFTATBgoJkiaJk/IsZAEZFgVDRFNZUzER
|
|
4
|
+
MA8GA1UEAxMIQ0RTWVMtQ0EwIBcNMTkwNzI2MDUxMjMwWhgPMjExODA3MjYwNTIy
|
|
5
|
+
MzBaMEExFTATBgoJkiaJk/IsZAEZFgVMT0NBTDEVMBMGCgmSJomT8ixkARkWBUNE
|
|
6
|
+
U1lTMREwDwYDVQQDEwhDRFNZUy1DQTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCC
|
|
7
|
+
AQoCggEBALjeLuRLJomO6Pojl8XYgK7aSVkY+fv34nK5Op1I/29U/FE3gMLm0nE3
|
|
8
|
+
gRGInG/a2EPouBRaW+vHpdakHQnJBI2VuO8AwCL3SVy2yuMq4Axa5GJPaqVthVCq
|
|
9
|
+
r8jHj4lY+Ybpy2JroQ5LIJfSvBEdGNgLIb5OIIJaA3DF44Llu7rJBo2t65pHrda8
|
|
10
|
+
uENImtikwhy9roA31iiOQKq4ccvF+E0+WLzLHd7ngHGHpTJsy7op+GQjZOsoV8ED
|
|
11
|
+
waHSCnGyh7FNnbZfYYFR6/aiFcXtBahJLpPe0jvyTR2JTS3HTeSXzVlUvXVIYxLb
|
|
12
|
+
jejezeRKv+CuNe3oh5AL57/BmJgumRUCAwEAAaNpMGcwEwYJKwYBBAGCNxQCBAYe
|
|
13
|
+
BABDAEEwDgYDVR0PAQH/BAQDAgGGMA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYE
|
|
14
|
+
FHH/eA+3icaHbj3eBS2KpGmte+MhMBAGCSsGAQQBgjcVAQQDAgEAMA0GCSqGSIb3
|
|
15
|
+
DQEBCwUAA4IBAQCCPLf7VPrubyTQ21YryKGvtDC5s7wkIaJo978SrSRMGotZBeoJ
|
|
16
|
+
e9bjxuH2MOhqcANT9Edx7nZfTkc1sebS4M6PW79P0zBlvWL0EbJslCerliyTZu7k
|
|
17
|
+
W55UfyEndcK12j3i4GLuklmXCAXwpFbvSsSW/NhMkY36VAO3iGd0rgk+FZWgh65Y
|
|
18
|
+
P+9FCKH0miXekJzAlxsC77cLG6VB8IiT+jz68MkTRBt1bIXr7dd2BQVQLeL8K8PC
|
|
19
|
+
thH4v8SNZgdf4DanMXjySLWvj7UrH+0HgOHjKFZo8axXTQOh2oz+gk2bTNju3jP9
|
|
20
|
+
trISLHBygmjaT5VyNNnRb45IoeXCrte5HMi7
|
|
21
|
+
-----END CERTIFICATE-----
|
|
22
|
+
-----BEGIN CERTIFICATE-----
|
|
23
|
+
MIIGkTCCBXmgAwIBAgITVQAAAvIbmEuSWZ/3sAAAAAAC8jANBgkqhkiG9w0BAQsF
|
|
24
|
+
ADBBMRUwEwYKCZImiZPyLGQBGRYFTE9DQUwxFTATBgoJkiaJk/IsZAEZFgVDRFNZ
|
|
25
|
+
UzERMA8GA1UEAxMIQ0RTWVMtQ0EwHhcNMjYwNTE4MTIzNDI5WhcNMjcwNTE4MTI0
|
|
26
|
+
NDI5WjCBhzELMAkGA1UEBhMCSU4xEDAOBgNVBAgTB0d1amFyYXQxEjAQBgNVBAcT
|
|
27
|
+
CUFobWVkYWJhZDESMBAGA1UEChMJQ3Jlc3REYXRhMR8wHQYDVQQLExZJbmZvcm1h
|
|
28
|
+
dGlvbiBUZWNobm9sb2d5MR0wGwYDVQQDExRhcGlwcm94eS5jZHN5cy5sb2NhbDCC
|
|
29
|
+
AiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBALJaNBpJuncXzdv/VFagnmNS
|
|
30
|
+
Sh/ArM+ZrvScI36w8jusjzDrJzHLU1VqpHUthRjmkpD3LG1xBAL4RQnc6dlPuyeg
|
|
31
|
+
e1B4TimyFoIk6ZkrPjBvDECNigFbZdZDyRbc/4q3yGPSwFqC4mZfVYuy4rbNFeBz
|
|
32
|
+
Qhbg7rYAppYxN8rV+F8GXoYSLL4hKQiaGdhi8dU3V1rkOXFjsSKlysCa8TBI8Ngn
|
|
33
|
+
bopAvD2bu3QMF6O5OeQlGiY/f4S9RGenPXPRUAc/xPYHD4LKdVs0kYPNhB6wYNoz
|
|
34
|
+
9pEM3uxwsBrjyLhplhQQTY/3stHhzq5TKYpe+DXfIhZqeczK/iHOtFbb4kteKWg4
|
|
35
|
+
hePdE68UqW1abqWPpt1Uqe2QXBfJ/HlRkuyJ074uynXWr/D6+6U2MhjxeOvA7qKr
|
|
36
|
+
AyeIKtEp/ZOnjXyj9kAsqkcYH8DY94U6NPu1wjvVFTc9H3p95+7Nldcy0rWnyFTp
|
|
37
|
+
gdNPwgj7FhTynYoPlaznsuyCMwTI4rXIvNA4eCFYFuPUKR2xpoWEh3itAcY0Y4Ar
|
|
38
|
+
GEBxIAHXNvXMIakLcKsGd6YShOMBWnfvVuhobyFEdPKfXIU8JS3MedFvsKgWkPLT
|
|
39
|
+
/JnkoDcwxCxIgBtMx1FaRTpIOzvhjjjExKUuyjYaWh3ChJVLaZHs3QUCuAm9jDol
|
|
40
|
+
3iy2IvQxuOIVuiYZNnaLAgMBAAGjggI5MIICNTAOBgNVHQ8BAf8EBAMCBaAwEwYD
|
|
41
|
+
VR0lBAwwCgYIKwYBBQUHAwEwJQYDVR0RBB4wHIIUYXBpcHJveHkuY2RzeXMubG9j
|
|
42
|
+
YWyHBAoyBr8wHQYDVR0OBBYEFCHwfcUcIpT12WQbt/Dalved8TIoMB8GA1UdIwQY
|
|
43
|
+
MBaAFHH/eA+3icaHbj3eBS2KpGmte+MhMIHGBgNVHR8Egb4wgbswgbiggbWggbKG
|
|
44
|
+
ga9sZGFwOi8vL0NOPUNEU1lTLUNBLENOPUNEU0RDMDEsQ049Q0RQLENOPVB1Ymxp
|
|
45
|
+
YyUyMEtleSUyMFNlcnZpY2VzLENOPVNlcnZpY2VzLENOPUNvbmZpZ3VyYXRpb24s
|
|
46
|
+
REM9Q0RTWVMsREM9TE9DQUw/Y2VydGlmaWNhdGVSZXZvY2F0aW9uTGlzdD9iYXNl
|
|
47
|
+
P29iamVjdENsYXNzPWNSTERpc3RyaWJ1dGlvblBvaW50MIG6BggrBgEFBQcBAQSB
|
|
48
|
+
rTCBqjCBpwYIKwYBBQUHMAKGgZpsZGFwOi8vL0NOPUNEU1lTLUNBLENOPUFJQSxD
|
|
49
|
+
Tj1QdWJsaWMlMjBLZXklMjBTZXJ2aWNlcyxDTj1TZXJ2aWNlcyxDTj1Db25maWd1
|
|
50
|
+
cmF0aW9uLERDPUNEU1lTLERDPUxPQ0FMP2NBQ2VydGlmaWNhdGU/YmFzZT9vYmpl
|
|
51
|
+
Y3RDbGFzcz1jZXJ0aWZpY2F0aW9uQXV0aG9yaXR5MCEGCSsGAQQBgjcUAgQUHhIA
|
|
52
|
+
VwBlAGIAUwBlAHIAdgBlAHIwDQYJKoZIhvcNAQELBQADggEBAFqh5t3037Tj8e/2
|
|
53
|
+
+5bwi/o3LQbNS4YXtmXYLu5rMXCJMaXthp9ypjMQTB6OP3m2R+zQx4rgBYevH0Ju
|
|
54
|
+
sjnw09w/571EsMY+Oh8Ehd3qwzfx+QxJ8loIsjaDE7xBnuzKxjXLY+Ritkt2yDmb
|
|
55
|
+
fgTCHMxIXJu4riFsMNuwwd7j6HOBP4pmymmhmFlO/kRtd7gn87eZlEfNSBNNq5pa
|
|
56
|
+
WLVNW0Jg1jrLAQwQfPe9mX5ID9xC9n46vSnbDukXR380zc9N55UrkpIHeU+VMQZh
|
|
57
|
+
AknBywOdKV1gFnwj2ItZT4wtLX42mXkCtH1KuJ0+LC2yLrswd0jT/Bpnimg0lyjh
|
|
58
|
+
AuEDBcQ=
|
|
59
|
+
-----END CERTIFICATE-----
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
-----BEGIN CERTIFICATE-----
|
|
2
|
+
MIIDdzCCAl+gAwIBAgIQOxQW8dLPJLdAoIx8ZgQzVzANBgkqhkiG9w0BAQsFADBB
|
|
3
|
+
MRUwEwYKCZImiZPyLGQBGRYFTE9DQUwxFTATBgoJkiaJk/IsZAEZFgVDRFNZUzER
|
|
4
|
+
MA8GA1UEAxMIQ0RTWVMtQ0EwIBcNMTkwNzI2MDUxMjMwWhgPMjExODA3MjYwNTIy
|
|
5
|
+
MzBaMEExFTATBgoJkiaJk/IsZAEZFgVMT0NBTDEVMBMGCgmSJomT8ixkARkWBUNE
|
|
6
|
+
U1lTMREwDwYDVQQDEwhDRFNZUy1DQTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCC
|
|
7
|
+
AQoCggEBALjeLuRLJomO6Pojl8XYgK7aSVkY+fv34nK5Op1I/29U/FE3gMLm0nE3
|
|
8
|
+
gRGInG/a2EPouBRaW+vHpdakHQnJBI2VuO8AwCL3SVy2yuMq4Axa5GJPaqVthVCq
|
|
9
|
+
r8jHj4lY+Ybpy2JroQ5LIJfSvBEdGNgLIb5OIIJaA3DF44Llu7rJBo2t65pHrda8
|
|
10
|
+
uENImtikwhy9roA31iiOQKq4ccvF+E0+WLzLHd7ngHGHpTJsy7op+GQjZOsoV8ED
|
|
11
|
+
waHSCnGyh7FNnbZfYYFR6/aiFcXtBahJLpPe0jvyTR2JTS3HTeSXzVlUvXVIYxLb
|
|
12
|
+
jejezeRKv+CuNe3oh5AL57/BmJgumRUCAwEAAaNpMGcwEwYJKwYBBAGCNxQCBAYe
|
|
13
|
+
BABDAEEwDgYDVR0PAQH/BAQDAgGGMA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYE
|
|
14
|
+
FHH/eA+3icaHbj3eBS2KpGmte+MhMBAGCSsGAQQBgjcVAQQDAgEAMA0GCSqGSIb3
|
|
15
|
+
DQEBCwUAA4IBAQCCPLf7VPrubyTQ21YryKGvtDC5s7wkIaJo978SrSRMGotZBeoJ
|
|
16
|
+
e9bjxuH2MOhqcANT9Edx7nZfTkc1sebS4M6PW79P0zBlvWL0EbJslCerliyTZu7k
|
|
17
|
+
W55UfyEndcK12j3i4GLuklmXCAXwpFbvSsSW/NhMkY36VAO3iGd0rgk+FZWgh65Y
|
|
18
|
+
P+9FCKH0miXekJzAlxsC77cLG6VB8IiT+jz68MkTRBt1bIXr7dd2BQVQLeL8K8PC
|
|
19
|
+
thH4v8SNZgdf4DanMXjySLWvj7UrH+0HgOHjKFZo8axXTQOh2oz+gk2bTNju3jP9
|
|
20
|
+
trISLHBygmjaT5VyNNnRb45IoeXCrte5HMi7
|
|
21
|
+
-----END CERTIFICATE-----
|
|
@@ -0,0 +1,230 @@
|
|
|
1
|
+
{
|
|
2
|
+
"comment": "Golden set for STEP CREATION only (planner accuracy, no binding). Each case gives the expected plan structure: step count, allowed verbs per step (synonyms welcome), dependency edges, repeat, agent_native. Grow this file from production misses logged via PLANWISE_PLAN_LOG_PATH — every failed route becomes a case here.",
|
|
3
|
+
"cases": [
|
|
4
|
+
{
|
|
5
|
+
"query": "Send an email to bob@corp.com saying the deploy is done",
|
|
6
|
+
"tags": ["single-step"],
|
|
7
|
+
"steps": [
|
|
8
|
+
{"verbs": ["send", "email"], "depends_on": [], "repeat": 1, "agent_native": false}
|
|
9
|
+
]
|
|
10
|
+
},
|
|
11
|
+
{
|
|
12
|
+
"query": "Fetch the user profile for user U-1234",
|
|
13
|
+
"tags": ["single-step"],
|
|
14
|
+
"steps": [
|
|
15
|
+
{"verbs": ["fetch", "get", "retrieve", "look", "lookup"], "depends_on": [], "repeat": 1, "agent_native": false}
|
|
16
|
+
]
|
|
17
|
+
},
|
|
18
|
+
{
|
|
19
|
+
"query": "Please list unpaid invoices for last month",
|
|
20
|
+
"tags": ["single-step", "pleasantry"],
|
|
21
|
+
"steps": [
|
|
22
|
+
{"verbs": ["list", "get", "fetch", "retrieve"], "depends_on": [], "repeat": 1, "agent_native": false}
|
|
23
|
+
]
|
|
24
|
+
},
|
|
25
|
+
{
|
|
26
|
+
"query": "Could you insert these rows into the orders table",
|
|
27
|
+
"tags": ["single-step", "pleasantry"],
|
|
28
|
+
"steps": [
|
|
29
|
+
{"verbs": ["insert", "write", "add"], "depends_on": [], "repeat": 1, "agent_native": false}
|
|
30
|
+
]
|
|
31
|
+
},
|
|
32
|
+
{
|
|
33
|
+
"query": "List unpaid invoices for last month and then send an email to boss@corp.com with the summary",
|
|
34
|
+
"tags": ["chain"],
|
|
35
|
+
"steps": [
|
|
36
|
+
{"verbs": ["list", "get", "fetch", "retrieve"], "depends_on": [], "repeat": 1, "agent_native": false},
|
|
37
|
+
{"verbs": ["send", "email"], "depends_on": [0], "repeat": 1, "agent_native": false}
|
|
38
|
+
]
|
|
39
|
+
},
|
|
40
|
+
{
|
|
41
|
+
"query": "Reset the password for user U-4455 and email them at kim@corp.com to confirm",
|
|
42
|
+
"tags": ["chain", "bare-and-split"],
|
|
43
|
+
"steps": [
|
|
44
|
+
{"verbs": ["reset"], "depends_on": [], "repeat": 1, "agent_native": false},
|
|
45
|
+
{"verbs": ["email", "send"], "depends_on": [0], "repeat": 1, "agent_native": false}
|
|
46
|
+
]
|
|
47
|
+
},
|
|
48
|
+
{
|
|
49
|
+
"query": "Create a jira ticket in ENG titled 'Fix login bug' and then post a message to the #eng channel about it",
|
|
50
|
+
"tags": ["chain"],
|
|
51
|
+
"steps": [
|
|
52
|
+
{"verbs": ["create", "add", "open"], "depends_on": [], "repeat": 1, "agent_native": false},
|
|
53
|
+
{"verbs": ["post", "send", "message"], "depends_on": [0], "repeat": 1, "agent_native": false}
|
|
54
|
+
]
|
|
55
|
+
},
|
|
56
|
+
{
|
|
57
|
+
"query": "Schedule a meeting with alice@corp.com and bob@corp.com tomorrow at 10am",
|
|
58
|
+
"tags": ["false-split", "entity-list"],
|
|
59
|
+
"steps": [
|
|
60
|
+
{"verbs": ["schedule", "create", "book"], "depends_on": [], "repeat": 1, "agent_native": false}
|
|
61
|
+
]
|
|
62
|
+
},
|
|
63
|
+
{
|
|
64
|
+
"query": "Get the purchase history of user U-9876",
|
|
65
|
+
"tags": ["single-step"],
|
|
66
|
+
"steps": [
|
|
67
|
+
{"verbs": ["get", "fetch", "retrieve"], "depends_on": [], "repeat": 1, "agent_native": false}
|
|
68
|
+
]
|
|
69
|
+
},
|
|
70
|
+
{
|
|
71
|
+
"query": "Search documents for the refund policy and summarize what you find",
|
|
72
|
+
"tags": ["chain", "agent-native"],
|
|
73
|
+
"steps": [
|
|
74
|
+
{"verbs": ["search", "find", "get", "retrieve"], "depends_on": [], "repeat": 1, "agent_native": false},
|
|
75
|
+
{"verbs": ["summarize", "summarise"], "depends_on": [0], "repeat": 1, "agent_native": true}
|
|
76
|
+
]
|
|
77
|
+
},
|
|
78
|
+
{
|
|
79
|
+
"query": "Pull last quarter's revenue and email the chart to finance@corp.com",
|
|
80
|
+
"tags": ["hidden-step"],
|
|
81
|
+
"steps": [
|
|
82
|
+
{"verbs": ["pull", "get", "fetch", "retrieve"], "depends_on": [], "repeat": 1, "agent_native": false},
|
|
83
|
+
{"verbs": ["generate", "create", "make"], "depends_on": [0], "repeat": 1, "agent_native": false},
|
|
84
|
+
{"verbs": ["email", "send"], "depends_on": [1], "repeat": 1, "agent_native": false}
|
|
85
|
+
]
|
|
86
|
+
},
|
|
87
|
+
{
|
|
88
|
+
"query": "Email the invoice summary to each of the account managers",
|
|
89
|
+
"tags": ["per-entity"],
|
|
90
|
+
"steps": [
|
|
91
|
+
{"verbs": ["email", "send"], "depends_on": [], "repeat": "per_entity", "agent_native": false}
|
|
92
|
+
]
|
|
93
|
+
},
|
|
94
|
+
{
|
|
95
|
+
"query": "Get the open tickets for every project and summarize them",
|
|
96
|
+
"tags": ["per-entity", "agent-native", "chain"],
|
|
97
|
+
"steps": [
|
|
98
|
+
{"verbs": ["get", "fetch", "list", "retrieve"], "depends_on": [], "repeat": "per_entity", "agent_native": false},
|
|
99
|
+
{"verbs": ["summarize", "summarise"], "depends_on": [0], "repeat": 1, "agent_native": true}
|
|
100
|
+
]
|
|
101
|
+
},
|
|
102
|
+
{
|
|
103
|
+
"query": "Compare the Q1 and Q2 revenue reports and recommend where to cut costs",
|
|
104
|
+
"tags": ["agent-native", "chain"],
|
|
105
|
+
"steps": [
|
|
106
|
+
{"verbs": ["get", "fetch", "pull", "retrieve"], "depends_on": [], "repeat": 2, "agent_native": false},
|
|
107
|
+
{"verbs": ["compare", "analyze", "analyse"], "depends_on": [0], "repeat": 1, "agent_native": true},
|
|
108
|
+
{"verbs": ["recommend", "decide", "explain"], "depends_on": [1], "repeat": 1, "agent_native": true}
|
|
109
|
+
]
|
|
110
|
+
},
|
|
111
|
+
{
|
|
112
|
+
"query": "Look up the profiles for alice and bob",
|
|
113
|
+
"tags": ["false-split", "entity-list"],
|
|
114
|
+
"steps": [
|
|
115
|
+
{"verbs": ["look", "lookup", "get", "fetch", "retrieve"], "depends_on": [], "repeat": 1, "agent_native": false}
|
|
116
|
+
]
|
|
117
|
+
},
|
|
118
|
+
{
|
|
119
|
+
"query": "Cancel my 3pm meeting and notify the attendees",
|
|
120
|
+
"tags": ["chain", "bare-and-split"],
|
|
121
|
+
"steps": [
|
|
122
|
+
{"verbs": ["cancel", "delete", "remove"], "depends_on": [], "repeat": 1, "agent_native": false},
|
|
123
|
+
{"verbs": ["notify", "send", "email", "message"], "depends_on": [0], "repeat": 1, "agent_native": false}
|
|
124
|
+
]
|
|
125
|
+
},
|
|
126
|
+
{
|
|
127
|
+
"query": "Download the sales spreadsheet, convert it to PDF, and upload it to the shared drive",
|
|
128
|
+
"tags": ["chain", "three-step"],
|
|
129
|
+
"steps": [
|
|
130
|
+
{"verbs": ["download", "get", "fetch"], "depends_on": [], "repeat": 1, "agent_native": false},
|
|
131
|
+
{"verbs": ["convert", "export", "generate"], "depends_on": [0], "repeat": 1, "agent_native": false},
|
|
132
|
+
{"verbs": ["upload", "write", "add"], "depends_on": [1], "repeat": 1, "agent_native": false}
|
|
133
|
+
]
|
|
134
|
+
},
|
|
135
|
+
{
|
|
136
|
+
"query": "Write these rows into the customers table, overwriting existing entries",
|
|
137
|
+
"tags": ["single-step"],
|
|
138
|
+
"steps": [
|
|
139
|
+
{"verbs": ["write", "insert", "update", "overwrite", "upsert"], "depends_on": [], "repeat": 1, "agent_native": false}
|
|
140
|
+
]
|
|
141
|
+
},
|
|
142
|
+
{
|
|
143
|
+
"query": "Find the invoice for order O-7788 and archive it",
|
|
144
|
+
"tags": ["chain", "bare-and-split", "pronoun"],
|
|
145
|
+
"steps": [
|
|
146
|
+
{"verbs": ["find", "get", "search", "fetch", "retrieve"], "depends_on": [], "repeat": 1, "agent_native": false},
|
|
147
|
+
{"verbs": ["archive"], "depends_on": [0], "repeat": 1, "agent_native": false}
|
|
148
|
+
]
|
|
149
|
+
},
|
|
150
|
+
{
|
|
151
|
+
"query": "Explain the difference between the staging and production configs",
|
|
152
|
+
"tags": ["agent-native", "hidden-step"],
|
|
153
|
+
"steps": [
|
|
154
|
+
{"verbs": ["get", "fetch", "read", "retrieve"], "depends_on": [], "repeat": 2, "agent_native": false},
|
|
155
|
+
{"verbs": ["explain", "compare", "describe"], "depends_on": [0], "repeat": 1, "agent_native": true}
|
|
156
|
+
]
|
|
157
|
+
},
|
|
158
|
+
{
|
|
159
|
+
"query": "Send the weekly status update to every regional manager",
|
|
160
|
+
"tags": ["per-entity", "held-out"],
|
|
161
|
+
"steps": [
|
|
162
|
+
{"verbs": ["send", "email", "message"], "depends_on": [], "repeat": "per_entity", "agent_native": false}
|
|
163
|
+
]
|
|
164
|
+
},
|
|
165
|
+
{
|
|
166
|
+
"query": "Pull the error logs for every service and explain the top failure",
|
|
167
|
+
"tags": ["per-entity", "agent-native", "chain", "held-out"],
|
|
168
|
+
"steps": [
|
|
169
|
+
{"verbs": ["pull", "get", "fetch", "retrieve"], "depends_on": [], "repeat": "per_entity", "agent_native": false},
|
|
170
|
+
{"verbs": ["explain", "summarize", "summarise", "describe"], "depends_on": [0], "repeat": 1, "agent_native": true}
|
|
171
|
+
]
|
|
172
|
+
},
|
|
173
|
+
{
|
|
174
|
+
"query": "Compare the March and April expense reports and decide which team overspent",
|
|
175
|
+
"tags": ["agent-native", "chain", "hidden-step", "held-out"],
|
|
176
|
+
"steps": [
|
|
177
|
+
{"verbs": ["get", "fetch", "pull", "retrieve"], "depends_on": [], "repeat": 2, "agent_native": false},
|
|
178
|
+
{"verbs": ["compare", "analyze", "analyse"], "depends_on": [0], "repeat": 1, "agent_native": true},
|
|
179
|
+
{"verbs": ["decide", "recommend", "explain"], "depends_on": [1], "repeat": 1, "agent_native": true}
|
|
180
|
+
]
|
|
181
|
+
},
|
|
182
|
+
{
|
|
183
|
+
"query": "Disable MFA for user U-7777 and message #security the confirmation",
|
|
184
|
+
"tags": ["bare-and-split", "held-out"],
|
|
185
|
+
"steps": [
|
|
186
|
+
{"verbs": ["disable", "turn", "remove", "update", "reset"], "depends_on": [], "repeat": 1, "agent_native": false},
|
|
187
|
+
{"verbs": ["message", "post", "send", "notify"], "depends_on": [0], "repeat": 1, "agent_native": false}
|
|
188
|
+
]
|
|
189
|
+
},
|
|
190
|
+
{
|
|
191
|
+
"query": "Email the onboarding doc to raj and priya",
|
|
192
|
+
"tags": ["false-split", "entity-list", "held-out"],
|
|
193
|
+
"steps": [
|
|
194
|
+
{"verbs": ["email", "send"], "depends_on": [], "repeat": 1, "agent_native": false}
|
|
195
|
+
]
|
|
196
|
+
},
|
|
197
|
+
{
|
|
198
|
+
"query": "Get last month's signups and email the trend graph to growth@corp.com",
|
|
199
|
+
"tags": ["hidden-step", "held-out"],
|
|
200
|
+
"steps": [
|
|
201
|
+
{"verbs": ["get", "fetch", "pull", "retrieve"], "depends_on": [], "repeat": 1, "agent_native": false},
|
|
202
|
+
{"verbs": ["generate", "create", "make", "chart"], "depends_on": [0], "repeat": 1, "agent_native": false},
|
|
203
|
+
{"verbs": ["email", "send"], "depends_on": [1], "repeat": 1, "agent_native": false}
|
|
204
|
+
]
|
|
205
|
+
},
|
|
206
|
+
{
|
|
207
|
+
"query": "Read the Q3 OKR doc and recommend which goals to drop",
|
|
208
|
+
"tags": ["agent-native", "chain", "bare-and-split", "held-out"],
|
|
209
|
+
"steps": [
|
|
210
|
+
{"verbs": ["read", "get", "fetch", "search", "find"], "depends_on": [], "repeat": 1, "agent_native": false},
|
|
211
|
+
{"verbs": ["recommend", "decide", "suggest"], "depends_on": [0], "repeat": 1, "agent_native": true}
|
|
212
|
+
]
|
|
213
|
+
},
|
|
214
|
+
{
|
|
215
|
+
"query": "Send the existing quarterly report to leadership@corp.com",
|
|
216
|
+
"tags": ["no-prefetch", "single-step", "held-out"],
|
|
217
|
+
"steps": [
|
|
218
|
+
{"verbs": ["send", "email"], "depends_on": [], "repeat": 1, "agent_native": false}
|
|
219
|
+
]
|
|
220
|
+
},
|
|
221
|
+
{
|
|
222
|
+
"query": "Delete the duplicate customer record and notify the data team on slack",
|
|
223
|
+
"tags": ["chain", "bare-and-split", "held-out"],
|
|
224
|
+
"steps": [
|
|
225
|
+
{"verbs": ["delete", "remove"], "depends_on": [], "repeat": 1, "agent_native": false},
|
|
226
|
+
{"verbs": ["notify", "post", "send", "message"], "depends_on": [0], "repeat": 1, "agent_native": false}
|
|
227
|
+
]
|
|
228
|
+
}
|
|
229
|
+
]
|
|
230
|
+
}
|