applied-cli 0.5.64__tar.gz → 0.5.65__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {applied_cli-0.5.64 → applied_cli-0.5.65}/PKG-INFO +1 -1
- {applied_cli-0.5.64 → applied_cli-0.5.65}/applied_cli/cli.py +10 -1
- {applied_cli-0.5.64 → applied_cli-0.5.65}/applied_cli/client.py +3 -0
- {applied_cli-0.5.64 → applied_cli-0.5.65}/applied_cli/tools.py +49 -8
- {applied_cli-0.5.64 → applied_cli-0.5.65}/applied_cli.egg-info/PKG-INFO +1 -1
- {applied_cli-0.5.64 → applied_cli-0.5.65}/pyproject.toml +1 -1
- {applied_cli-0.5.64 → applied_cli-0.5.65}/README.md +0 -0
- {applied_cli-0.5.64 → applied_cli-0.5.65}/applied_cli/__init__.py +0 -0
- {applied_cli-0.5.64 → applied_cli-0.5.65}/applied_cli/agent_scoped_flows.py +0 -0
- {applied_cli-0.5.64 → applied_cli-0.5.65}/applied_cli/conversation_lookup.py +0 -0
- {applied_cli-0.5.64 → applied_cli-0.5.65}/applied_cli/conversations.py +0 -0
- {applied_cli-0.5.64 → applied_cli-0.5.65}/applied_cli/credentials.py +0 -0
- {applied_cli-0.5.64 → applied_cli-0.5.65}/applied_cli/flow_helpers.py +0 -0
- {applied_cli-0.5.64 → applied_cli-0.5.65}/applied_cli/formatters.py +0 -0
- {applied_cli-0.5.64 → applied_cli-0.5.65}/applied_cli.egg-info/SOURCES.txt +0 -0
- {applied_cli-0.5.64 → applied_cli-0.5.65}/applied_cli.egg-info/dependency_links.txt +0 -0
- {applied_cli-0.5.64 → applied_cli-0.5.65}/applied_cli.egg-info/entry_points.txt +0 -0
- {applied_cli-0.5.64 → applied_cli-0.5.65}/applied_cli.egg-info/requires.txt +0 -0
- {applied_cli-0.5.64 → applied_cli-0.5.65}/applied_cli.egg-info/top_level.txt +0 -0
- {applied_cli-0.5.64 → applied_cli-0.5.65}/setup.cfg +0 -0
- {applied_cli-0.5.64 → applied_cli-0.5.65}/tests/test_agent_scoped_flows.py +0 -0
- {applied_cli-0.5.64 → applied_cli-0.5.65}/tests/test_audit_tools.py +0 -0
- {applied_cli-0.5.64 → applied_cli-0.5.65}/tests/test_benchmark_scenario_tools.py +0 -0
- {applied_cli-0.5.64 → applied_cli-0.5.65}/tests/test_cli.py +0 -0
- {applied_cli-0.5.64 → applied_cli-0.5.65}/tests/test_client.py +0 -0
- {applied_cli-0.5.64 → applied_cli-0.5.65}/tests/test_conversation_tools.py +0 -0
- {applied_cli-0.5.64 → applied_cli-0.5.65}/tests/test_flow_tools.py +0 -0
|
@@ -1310,6 +1310,9 @@ def scenario_runs(
|
|
|
1310
1310
|
benchmark_id: str = typer.Option(
|
|
1311
1311
|
None, "--benchmark-id", help="Filter by benchmark ID"
|
|
1312
1312
|
),
|
|
1313
|
+
bulk_job_id: str = typer.Option(
|
|
1314
|
+
None, "--bulk-job-id", help="Filter by bulk run job ID"
|
|
1315
|
+
),
|
|
1313
1316
|
latest: bool = typer.Option(
|
|
1314
1317
|
True, "--latest/--all", help="Only show the latest run per scenario"
|
|
1315
1318
|
),
|
|
@@ -1325,6 +1328,7 @@ def scenario_runs(
|
|
|
1325
1328
|
client,
|
|
1326
1329
|
scenario_id=scenario_id,
|
|
1327
1330
|
benchmark_id=benchmark_id,
|
|
1331
|
+
bulk_job_id=bulk_job_id,
|
|
1328
1332
|
latest=latest,
|
|
1329
1333
|
output_format=format,
|
|
1330
1334
|
)
|
|
@@ -1403,6 +1407,9 @@ def scenario_bulk_run(
|
|
|
1403
1407
|
@app.command("scenario-bulk-status")
|
|
1404
1408
|
def scenario_bulk_status(
|
|
1405
1409
|
job_id: str = typer.Argument(..., help="Bulk run job ID"),
|
|
1410
|
+
include_runs: bool = typer.Option(
|
|
1411
|
+
False, "--include-runs", help="Include per-run details with scenario mappings"
|
|
1412
|
+
),
|
|
1406
1413
|
shop_id: str = typer.Option(None, "--shop-id", help="Override shop ID"),
|
|
1407
1414
|
format: str = typer.Option(
|
|
1408
1415
|
"text", "--format", "-f", help="Output format: text or json"
|
|
@@ -1411,7 +1418,9 @@ def scenario_bulk_status(
|
|
|
1411
1418
|
"""Get status for a bulk scenario rerun job."""
|
|
1412
1419
|
client = get_client(shop_id=shop_id)
|
|
1413
1420
|
result = asyncio.run(
|
|
1414
|
-
tools.scenario_bulk_status(
|
|
1421
|
+
tools.scenario_bulk_status(
|
|
1422
|
+
client, job_id=job_id, include_runs=include_runs, output_format=format
|
|
1423
|
+
)
|
|
1415
1424
|
)
|
|
1416
1425
|
typer.echo(result)
|
|
1417
1426
|
|
|
@@ -1850,6 +1850,7 @@ class AppliedClient:
|
|
|
1850
1850
|
self,
|
|
1851
1851
|
scenario_id: str | None = None,
|
|
1852
1852
|
benchmark_id: str | None = None,
|
|
1853
|
+
bulk_job_id: str | None = None,
|
|
1853
1854
|
latest: bool = False,
|
|
1854
1855
|
limit: int = 50,
|
|
1855
1856
|
) -> list[dict]:
|
|
@@ -1859,6 +1860,8 @@ class AppliedClient:
|
|
|
1859
1860
|
params["scenario_id"] = scenario_id
|
|
1860
1861
|
if benchmark_id:
|
|
1861
1862
|
params["benchmark_id"] = benchmark_id
|
|
1863
|
+
if bulk_job_id:
|
|
1864
|
+
params["bulk_job_id"] = bulk_job_id
|
|
1862
1865
|
if latest:
|
|
1863
1866
|
params["latest"] = "true"
|
|
1864
1867
|
data = await self._request("GET", "/v1/scenario-runs/", params=params)
|
|
@@ -200,7 +200,15 @@ def _project_scenario_payload(
|
|
|
200
200
|
|
|
201
201
|
|
|
202
202
|
def _project_scenario_run_payload(run: dict) -> dict:
|
|
203
|
-
scenario
|
|
203
|
+
# scenario can be a UUID string (PK) or a nested dict
|
|
204
|
+
scenario_raw = run.get("scenario")
|
|
205
|
+
if isinstance(scenario_raw, dict):
|
|
206
|
+
scenario_id = scenario_raw.get("id")
|
|
207
|
+
scenario_name = scenario_raw.get("name")
|
|
208
|
+
else:
|
|
209
|
+
scenario_id = scenario_raw
|
|
210
|
+
scenario_name = run.get("scenario_name")
|
|
211
|
+
|
|
204
212
|
evaluated_by = run.get("evaluated_by") or {}
|
|
205
213
|
return {
|
|
206
214
|
"id": run.get("id"),
|
|
@@ -219,8 +227,8 @@ def _project_scenario_run_payload(run: dict) -> dict:
|
|
|
219
227
|
"evaluated_by_id": evaluated_by.get("id"),
|
|
220
228
|
"evaluated_by_name": evaluated_by.get("name"),
|
|
221
229
|
"scenario": {
|
|
222
|
-
"id":
|
|
223
|
-
"name":
|
|
230
|
+
"id": scenario_id,
|
|
231
|
+
"name": scenario_name,
|
|
224
232
|
},
|
|
225
233
|
"output_conversation": _project_embedded_conversation(
|
|
226
234
|
run.get("output_conversation")
|
|
@@ -4786,6 +4794,7 @@ async def scenario_run_list(
|
|
|
4786
4794
|
client: AppliedClient,
|
|
4787
4795
|
scenario_id: str | None = None,
|
|
4788
4796
|
benchmark_id: str | None = None,
|
|
4797
|
+
bulk_job_id: str | None = None,
|
|
4789
4798
|
latest: bool = True,
|
|
4790
4799
|
output_format: str = "csv",
|
|
4791
4800
|
) -> str:
|
|
@@ -4796,19 +4805,30 @@ async def scenario_run_list(
|
|
|
4796
4805
|
client: Authenticated AppliedClient
|
|
4797
4806
|
scenario_id: Optional - filter by scenario UUID
|
|
4798
4807
|
benchmark_id: Optional - filter by benchmark UUID
|
|
4808
|
+
bulk_job_id: Optional - filter by bulk run job UUID
|
|
4799
4809
|
latest: If True, show only the most recent run per scenario (default True)
|
|
4800
4810
|
output_format: 'csv' or 'json'
|
|
4801
4811
|
|
|
4802
4812
|
Returns:
|
|
4803
|
-
List of scenario runs with id, scenario_name, pass_status,
|
|
4813
|
+
List of scenario runs with id, scenario_id, scenario_name, pass_status, output_conversation_id
|
|
4804
4814
|
"""
|
|
4805
4815
|
runs = await client.list_scenario_runs(
|
|
4806
|
-
scenario_id=scenario_id,
|
|
4816
|
+
scenario_id=scenario_id,
|
|
4817
|
+
benchmark_id=benchmark_id,
|
|
4818
|
+
bulk_job_id=bulk_job_id,
|
|
4819
|
+
latest=latest,
|
|
4807
4820
|
)
|
|
4821
|
+
def _extract_scenario_fields(r: dict) -> tuple:
|
|
4822
|
+
scenario_raw = r.get("scenario")
|
|
4823
|
+
if isinstance(scenario_raw, dict):
|
|
4824
|
+
return scenario_raw.get("id", ""), scenario_raw.get("name", "")
|
|
4825
|
+
return scenario_raw or "", r.get("scenario_name", "")
|
|
4826
|
+
|
|
4808
4827
|
mapped = [
|
|
4809
4828
|
{
|
|
4810
4829
|
"id": r.get("id"),
|
|
4811
|
-
"
|
|
4830
|
+
"scenario_id": _extract_scenario_fields(r)[0],
|
|
4831
|
+
"scenario_name": _extract_scenario_fields(r)[1],
|
|
4812
4832
|
"status": r.get("status"),
|
|
4813
4833
|
"pass_status": r.get("pass_status", "unrated"),
|
|
4814
4834
|
"csat_score": r.get("csat_score"),
|
|
@@ -4825,6 +4845,7 @@ async def scenario_run_list(
|
|
|
4825
4845
|
mapped,
|
|
4826
4846
|
[
|
|
4827
4847
|
"id",
|
|
4848
|
+
"scenario_id",
|
|
4828
4849
|
"scenario_name",
|
|
4829
4850
|
"status",
|
|
4830
4851
|
"pass_status",
|
|
@@ -4987,12 +5008,14 @@ async def scenario_bulk_run(
|
|
|
4987
5008
|
output += f"scenario_run_ids: {preview_ids}\n"
|
|
4988
5009
|
if len(run_ids) > 10:
|
|
4989
5010
|
output += f"more_runs: {len(run_ids) - 10}\n"
|
|
5011
|
+
output += f"\nTip: use scenario_bulk_status(job_id, include_runs=True) or scenario_run_list(bulk_job_id=job_id) to get per-run details with scenario mappings."
|
|
4990
5012
|
return output
|
|
4991
5013
|
|
|
4992
5014
|
|
|
4993
5015
|
async def scenario_bulk_status(
|
|
4994
5016
|
client: AppliedClient,
|
|
4995
5017
|
job_id: str,
|
|
5018
|
+
include_runs: bool = False,
|
|
4996
5019
|
output_format: str = "text",
|
|
4997
5020
|
) -> str:
|
|
4998
5021
|
"""
|
|
@@ -5001,10 +5024,11 @@ async def scenario_bulk_status(
|
|
|
5001
5024
|
Args:
|
|
5002
5025
|
client: Authenticated AppliedClient
|
|
5003
5026
|
job_id: The bulk run job UUID
|
|
5027
|
+
include_runs: If True, include per-run details (scenario_id, output_conversation_id, status)
|
|
5004
5028
|
output_format: 'text' (default) or 'json'
|
|
5005
5029
|
|
|
5006
5030
|
Returns:
|
|
5007
|
-
Counts, timing information, and
|
|
5031
|
+
Counts, timing information, failed run details, and optionally per-run details
|
|
5008
5032
|
"""
|
|
5009
5033
|
try:
|
|
5010
5034
|
result = await client.get_scenario_bulk_run_status(job_id)
|
|
@@ -5012,7 +5036,10 @@ async def scenario_bulk_status(
|
|
|
5012
5036
|
return _format_error(e)
|
|
5013
5037
|
|
|
5014
5038
|
if output_format == "json":
|
|
5015
|
-
|
|
5039
|
+
payload = dict(result)
|
|
5040
|
+
if not include_runs:
|
|
5041
|
+
payload.pop("runs", None)
|
|
5042
|
+
return to_json(payload)
|
|
5016
5043
|
|
|
5017
5044
|
counts = result.get("counts") or {}
|
|
5018
5045
|
output = "# Bulk Run Status\n"
|
|
@@ -5034,6 +5061,20 @@ async def scenario_bulk_status(
|
|
|
5034
5061
|
output += f"\n# Failed Runs ({len(failed)})\n"
|
|
5035
5062
|
output += to_json(failed)
|
|
5036
5063
|
|
|
5064
|
+
runs_detail = result.get("runs") or []
|
|
5065
|
+
if include_runs and runs_detail:
|
|
5066
|
+
output += f"\n\n# Per-Run Details ({len(runs_detail)})\n"
|
|
5067
|
+
output += to_csv(
|
|
5068
|
+
runs_detail,
|
|
5069
|
+
[
|
|
5070
|
+
"id",
|
|
5071
|
+
"scenario_id",
|
|
5072
|
+
"status",
|
|
5073
|
+
"pass_status",
|
|
5074
|
+
"output_conversation_id",
|
|
5075
|
+
],
|
|
5076
|
+
)
|
|
5077
|
+
|
|
5037
5078
|
return output
|
|
5038
5079
|
|
|
5039
5080
|
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|