typesafeai-cli 0.3.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- typesafe_cli/__init__.py +1 -0
- typesafe_cli/cli.py +93 -0
- typesafe_cli/client.py +106 -0
- typesafe_cli/commands/__init__.py +0 -0
- typesafe_cli/commands/agent.py +12 -0
- typesafe_cli/commands/ask.py +90 -0
- typesafe_cli/commands/auth.py +28 -0
- typesafe_cli/commands/decide.py +43 -0
- typesafe_cli/commands/eval.py +37 -0
- typesafe_cli/commands/extract.py +83 -0
- typesafe_cli/commands/find.py +53 -0
- typesafe_cli/commands/models.py +30 -0
- typesafe_cli/commands/oneshot.py +91 -0
- typesafe_cli/commands/rank.py +49 -0
- typesafe_cli/commands/screen.py +38 -0
- typesafe_cli/commands/skills.py +138 -0
- typesafe_cli/commands/smoke.py +35 -0
- typesafe_cli/commands/suggest_skill.py +48 -0
- typesafe_cli/commands/verify.py +53 -0
- typesafe_cli/config.py +37 -0
- typesafe_cli/data/__init__.py +0 -0
- typesafe_cli/data/typesafe-ai/CLI.md +7 -0
- typesafe_cli/data/typesafe-ai/SKILL.md +149 -0
- typesafe_cli/data/typesafe-cli/SKILL.md +137 -0
- typesafe_cli/detect.py +83 -0
- typesafe_cli/format.py +22 -0
- typesafe_cli/help.py +36 -0
- typesafe_cli/io.py +17 -0
- typesafe_cli/questions.py +96 -0
- typesafe_cli/recipes/__init__.py +3 -0
- typesafe_cli/recipes/decide.py +64 -0
- typesafe_cli/recipes/extract.py +42 -0
- typesafe_cli/recipes/find.py +90 -0
- typesafe_cli/recipes/rank.py +97 -0
- typesafe_cli/recipes/screen.py +68 -0
- typesafe_cli/recipes/skills.py +148 -0
- typesafe_cli/recipes/thresholds.py +24 -0
- typesafe_cli/recipes/verify.py +48 -0
- typesafe_cli/schema.py +150 -0
- typesafeai_cli-0.3.1.dist-info/METADATA +137 -0
- typesafeai_cli-0.3.1.dist-info/RECORD +44 -0
- typesafeai_cli-0.3.1.dist-info/WHEEL +4 -0
- typesafeai_cli-0.3.1.dist-info/entry_points.txt +2 -0
- typesafeai_cli-0.3.1.dist-info/licenses/LICENSE +21 -0
typesafe_cli/__init__.py
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
__version__ = "0.3.1"
|
typesafe_cli/cli.py
ADDED
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import typer
|
|
4
|
+
|
|
5
|
+
from typesafe_cli.commands.agent import schema as agent_schema
|
|
6
|
+
from typesafe_cli.commands.ask import ask
|
|
7
|
+
from typesafe_cli.commands.auth import status as auth_status
|
|
8
|
+
from typesafe_cli.commands.decide import decide
|
|
9
|
+
from typesafe_cli.commands.extract import extract
|
|
10
|
+
from typesafe_cli.commands.find import find
|
|
11
|
+
from typesafe_cli.commands.models import models
|
|
12
|
+
from typesafe_cli.commands.oneshot import choice, noul, score
|
|
13
|
+
from typesafe_cli.commands.rank import rank
|
|
14
|
+
from typesafe_cli.commands.screen import screen
|
|
15
|
+
from typesafe_cli.commands.skills import install as skills_install
|
|
16
|
+
from typesafe_cli.commands.skills import list_skills
|
|
17
|
+
from typesafe_cli.commands.smoke import smoke
|
|
18
|
+
from typesafe_cli.commands.suggest_skill import suggest_skill
|
|
19
|
+
from typesafe_cli.commands.verify import verify
|
|
20
|
+
from typesafe_cli.help import AgentCommand, AgentGroup
|
|
21
|
+
|
|
22
|
+
app = typer.Typer(
|
|
23
|
+
name="typesafe",
|
|
24
|
+
cls=AgentGroup,
|
|
25
|
+
no_args_is_help=True,
|
|
26
|
+
add_completion=False,
|
|
27
|
+
pretty_exceptions_enable=False,
|
|
28
|
+
help="Agent CLI for TypeSafe System One (Jev). Typed judgments, not chat. Agents must not read TYPESAFE_* env vars or key files; use typesafe auth status.",
|
|
29
|
+
)
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def _on_agent(args: bool = False) -> bool:
|
|
33
|
+
if args:
|
|
34
|
+
from typesafe_cli.detect import set_force_agent
|
|
35
|
+
|
|
36
|
+
set_force_agent(True)
|
|
37
|
+
return args
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def _on_no_agent(args: bool = False) -> bool:
|
|
41
|
+
if args:
|
|
42
|
+
from typesafe_cli.detect import set_force_agent
|
|
43
|
+
|
|
44
|
+
set_force_agent(False)
|
|
45
|
+
return args
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
@app.callback()
|
|
49
|
+
def _global_options(
|
|
50
|
+
agent: bool = typer.Option(
|
|
51
|
+
False,
|
|
52
|
+
"--agent",
|
|
53
|
+
help="Force agent mode (JSON help/schema)",
|
|
54
|
+
is_eager=True,
|
|
55
|
+
callback=_on_agent,
|
|
56
|
+
),
|
|
57
|
+
no_agent: bool = typer.Option(
|
|
58
|
+
False,
|
|
59
|
+
"--no-agent",
|
|
60
|
+
help="Disable agent mode",
|
|
61
|
+
is_eager=True,
|
|
62
|
+
callback=_on_no_agent,
|
|
63
|
+
),
|
|
64
|
+
) -> None:
|
|
65
|
+
del agent, no_agent
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
auth_app = typer.Typer(cls=AgentGroup, no_args_is_help=True, help="Authentication helpers")
|
|
69
|
+
auth_app.command("status", cls=AgentCommand)(auth_status)
|
|
70
|
+
app.add_typer(auth_app, name="auth")
|
|
71
|
+
|
|
72
|
+
agent_app = typer.Typer(cls=AgentGroup, no_args_is_help=True, help="Agent discovery")
|
|
73
|
+
agent_app.command("schema", cls=AgentCommand)(agent_schema)
|
|
74
|
+
app.add_typer(agent_app, name="agent")
|
|
75
|
+
|
|
76
|
+
skills_app = typer.Typer(cls=AgentGroup, no_args_is_help=True, help="Install the official TypeSafe skill")
|
|
77
|
+
skills_app.command("install", cls=AgentCommand)(skills_install)
|
|
78
|
+
skills_app.command("list", cls=AgentCommand)(list_skills)
|
|
79
|
+
app.add_typer(skills_app, name="skills")
|
|
80
|
+
|
|
81
|
+
app.command("ask", cls=AgentCommand)(ask)
|
|
82
|
+
app.command("noul", cls=AgentCommand)(noul)
|
|
83
|
+
app.command("choice", cls=AgentCommand)(choice)
|
|
84
|
+
app.command("score", cls=AgentCommand)(score)
|
|
85
|
+
app.command("find", cls=AgentCommand)(find)
|
|
86
|
+
app.command("rank", cls=AgentCommand)(rank)
|
|
87
|
+
app.command("extract", cls=AgentCommand)(extract)
|
|
88
|
+
app.command("verify", cls=AgentCommand)(verify)
|
|
89
|
+
app.command("screen", cls=AgentCommand)(screen)
|
|
90
|
+
app.command("suggest-skill", cls=AgentCommand)(suggest_skill)
|
|
91
|
+
app.command("decide", cls=AgentCommand)(decide)
|
|
92
|
+
app.command("models", cls=AgentCommand)(models)
|
|
93
|
+
app.command("smoke", cls=AgentCommand)(smoke)
|
typesafe_cli/client.py
ADDED
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from typing import Any
|
|
4
|
+
|
|
5
|
+
from typesafe_sdk import Choice, Noul, Score, TypeSafeClient
|
|
6
|
+
|
|
7
|
+
from typesafe_cli.config import Config
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
def answers_to_dict(response: Any) -> dict[str, Any]:
|
|
11
|
+
answers: dict[str, Any] = {}
|
|
12
|
+
raw_answers = getattr(response, "answers", {}) or {}
|
|
13
|
+
for qid, answer in raw_answers.items():
|
|
14
|
+
answers[qid] = _answer_to_dict(answer)
|
|
15
|
+
usage = getattr(response, "usage", None)
|
|
16
|
+
return {
|
|
17
|
+
"model": getattr(response, "model", None),
|
|
18
|
+
"answers": answers,
|
|
19
|
+
"usage": {
|
|
20
|
+
"input_tokens": getattr(usage, "input_tokens", None) if usage is not None else None,
|
|
21
|
+
"output_tokens": getattr(usage, "output_tokens", None) if usage is not None else None,
|
|
22
|
+
},
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def _answer_to_dict(answer: Any) -> dict[str, Any]:
|
|
27
|
+
if hasattr(answer, "noul") and not hasattr(answer, "choice") and not hasattr(answer, "score"):
|
|
28
|
+
return {"type": "noul", "noul": answer.noul}
|
|
29
|
+
if hasattr(answer, "choice"):
|
|
30
|
+
return {
|
|
31
|
+
"type": "choice",
|
|
32
|
+
"choice": answer.choice,
|
|
33
|
+
"probabilities": _stringify_keys(getattr(answer, "probabilities", {})),
|
|
34
|
+
"confidence": getattr(answer, "confidence", None),
|
|
35
|
+
}
|
|
36
|
+
if hasattr(answer, "score"):
|
|
37
|
+
return {
|
|
38
|
+
"type": "score",
|
|
39
|
+
"score": answer.score,
|
|
40
|
+
"legend": _stringify_keys(getattr(answer, "legend", {})),
|
|
41
|
+
"probabilities": _stringify_keys(getattr(answer, "probabilities", {})),
|
|
42
|
+
"confidence": getattr(answer, "confidence", None),
|
|
43
|
+
}
|
|
44
|
+
raise TypeError(f"unsupported answer type: {type(answer)!r}")
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def _stringify_keys(mapping: Any) -> dict[str, Any]:
|
|
48
|
+
if not mapping:
|
|
49
|
+
return {}
|
|
50
|
+
return {str(key): value for key, value in dict(mapping).items()}
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def to_sdk_questions(questions: dict[str, dict[str, Any]]) -> dict[str, Any]:
|
|
54
|
+
sdk: dict[str, Any] = {}
|
|
55
|
+
for qid, question in questions.items():
|
|
56
|
+
qtype = question["type"]
|
|
57
|
+
instructions = question["instructions"]
|
|
58
|
+
criteria = question.get("criteria")
|
|
59
|
+
if qtype == "noul":
|
|
60
|
+
sdk[qid] = Noul(instructions=instructions, criteria=criteria)
|
|
61
|
+
elif qtype == "choice":
|
|
62
|
+
sdk[qid] = Choice(instructions=instructions, criteria=criteria)
|
|
63
|
+
elif qtype == "score":
|
|
64
|
+
sdk[qid] = Score(instructions=instructions, criteria=criteria)
|
|
65
|
+
else:
|
|
66
|
+
raise ValueError(f"unknown question type: {qtype}")
|
|
67
|
+
return sdk
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def system_one(
|
|
71
|
+
*,
|
|
72
|
+
config: Config,
|
|
73
|
+
state: str | dict | list,
|
|
74
|
+
questions: dict[str, dict[str, Any]],
|
|
75
|
+
model: str | None = None,
|
|
76
|
+
) -> dict[str, Any]:
|
|
77
|
+
if not config.api_key:
|
|
78
|
+
raise RuntimeError("missing API key")
|
|
79
|
+
resolved_model = model or config.model
|
|
80
|
+
with TypeSafeClient(
|
|
81
|
+
api_key=config.api_key,
|
|
82
|
+
base_url=config.base_url,
|
|
83
|
+
model=resolved_model,
|
|
84
|
+
) as client:
|
|
85
|
+
response = client.system_one(
|
|
86
|
+
state=state,
|
|
87
|
+
questions=to_sdk_questions(questions),
|
|
88
|
+
model=resolved_model,
|
|
89
|
+
)
|
|
90
|
+
return answers_to_dict(response)
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
def list_models(*, config: Config) -> dict[str, Any]:
|
|
94
|
+
if not config.api_key:
|
|
95
|
+
raise RuntimeError("missing API key")
|
|
96
|
+
with TypeSafeClient(api_key=config.api_key, base_url=config.base_url) as client:
|
|
97
|
+
listed = client.models.list()
|
|
98
|
+
models = [
|
|
99
|
+
{
|
|
100
|
+
"name": getattr(item, "name", None),
|
|
101
|
+
"description": getattr(item, "description", None),
|
|
102
|
+
"release_date": getattr(item, "release_date", None),
|
|
103
|
+
}
|
|
104
|
+
for item in getattr(listed, "models", ())
|
|
105
|
+
]
|
|
106
|
+
return {"models": models}
|
|
File without changes
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import typer
|
|
4
|
+
|
|
5
|
+
from typesafe_cli.io import emit_success
|
|
6
|
+
from typesafe_cli.schema import command_schema
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def schema(
|
|
10
|
+
compact: bool = typer.Option(False, "--compact", help="Names and flags only"),
|
|
11
|
+
) -> None:
|
|
12
|
+
emit_success(data=command_schema(compact=compact), command="agent schema")
|
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import sys
|
|
4
|
+
from pathlib import Path
|
|
5
|
+
from typing import Any
|
|
6
|
+
|
|
7
|
+
import typer
|
|
8
|
+
from typesafe_sdk import TypeSafeError
|
|
9
|
+
|
|
10
|
+
from typesafe_cli.client import system_one
|
|
11
|
+
from typesafe_cli.config import load_config
|
|
12
|
+
from typesafe_cli.io import emit_success, fail
|
|
13
|
+
from typesafe_cli.questions import QuestionError, load_questions_file, load_state_file, parse_state_text
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def ask(
|
|
17
|
+
state: str | None = typer.Option(None, "--state", help="Inline state (string or JSON)"),
|
|
18
|
+
state_file: Path | None = typer.Option(None, "--state-file", help="State file; '-' reads stdin"),
|
|
19
|
+
questions_file: Path | None = typer.Option(None, "--questions-file", help="JSON questions file"),
|
|
20
|
+
model: str | None = typer.Option(None, "--model"),
|
|
21
|
+
key: str | None = typer.Option(None, "--key"),
|
|
22
|
+
creds: Path | None = typer.Option(None, "--creds"),
|
|
23
|
+
) -> None:
|
|
24
|
+
if questions_file is None:
|
|
25
|
+
fail(code="usage", message="--questions-file is required", exit_code=2)
|
|
26
|
+
|
|
27
|
+
try:
|
|
28
|
+
questions, embedded_state = load_questions_file(questions_file)
|
|
29
|
+
resolved_state = _resolve_state(state=state, state_file=state_file, embedded=embedded_state)
|
|
30
|
+
config = load_config(key=key, creds=creds, model=model)
|
|
31
|
+
except QuestionError as exc:
|
|
32
|
+
fail(code="usage", message=str(exc), exit_code=2)
|
|
33
|
+
except (OSError, ValueError) as exc:
|
|
34
|
+
fail(code="usage", message=str(exc), exit_code=2)
|
|
35
|
+
|
|
36
|
+
if not config.api_key:
|
|
37
|
+
fail(code="auth", message="missing TYPESAFE_API_KEY (or --key / --creds)", exit_code=1)
|
|
38
|
+
|
|
39
|
+
try:
|
|
40
|
+
data = system_one(config=config, state=resolved_state, questions=questions, model=model)
|
|
41
|
+
except TypeSafeError as exc:
|
|
42
|
+
fail(code="request", message=str(exc), exit_code=1)
|
|
43
|
+
except Exception as exc: # noqa: BLE001 — surface SDK/network failures as exit 1
|
|
44
|
+
fail(code="request", message=str(exc), exit_code=1)
|
|
45
|
+
|
|
46
|
+
emit_success(data=data, command="ask")
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def _resolve_state(
|
|
50
|
+
*,
|
|
51
|
+
state: str | None,
|
|
52
|
+
state_file: Path | None,
|
|
53
|
+
embedded: str | dict | list | None,
|
|
54
|
+
) -> str | dict | list:
|
|
55
|
+
provided = int(state is not None) + int(state_file is not None)
|
|
56
|
+
if provided > 1:
|
|
57
|
+
raise QuestionError("use only one of --state or --state-file")
|
|
58
|
+
if state is not None:
|
|
59
|
+
return parse_state_text(state)
|
|
60
|
+
if state_file is not None:
|
|
61
|
+
if str(state_file) == "-":
|
|
62
|
+
return parse_state_text(sys.stdin.read())
|
|
63
|
+
return load_state_file(state_file)
|
|
64
|
+
if embedded is not None:
|
|
65
|
+
return embedded
|
|
66
|
+
raise QuestionError("state is required (--state, --state-file, or state in the questions file)")
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def run_evaluation(
|
|
70
|
+
*,
|
|
71
|
+
state: str | dict | list,
|
|
72
|
+
questions: dict[str, dict[str, Any]],
|
|
73
|
+
key: str | None,
|
|
74
|
+
creds: Path | None,
|
|
75
|
+
model: str | None,
|
|
76
|
+
command: str,
|
|
77
|
+
) -> None:
|
|
78
|
+
try:
|
|
79
|
+
config = load_config(key=key, creds=creds, model=model)
|
|
80
|
+
except (OSError, ValueError) as exc:
|
|
81
|
+
fail(code="usage", message=str(exc), exit_code=2)
|
|
82
|
+
if not config.api_key:
|
|
83
|
+
fail(code="auth", message="missing TYPESAFE_API_KEY (or --key / --creds)", exit_code=1)
|
|
84
|
+
try:
|
|
85
|
+
data = system_one(config=config, state=state, questions=questions, model=model)
|
|
86
|
+
except TypeSafeError as exc:
|
|
87
|
+
fail(code="request", message=str(exc), exit_code=1)
|
|
88
|
+
except Exception as exc: # noqa: BLE001
|
|
89
|
+
fail(code="request", message=str(exc), exit_code=1)
|
|
90
|
+
emit_success(data=data, command=command)
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from pathlib import Path
|
|
4
|
+
|
|
5
|
+
import typer
|
|
6
|
+
|
|
7
|
+
from typesafe_cli.config import load_config
|
|
8
|
+
from typesafe_cli.io import emit_success, fail
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def status(
|
|
12
|
+
key: str | None = typer.Option(None, "--key"),
|
|
13
|
+
creds: Path | None = typer.Option(None, "--creds"),
|
|
14
|
+
model: str | None = typer.Option(None, "--model"),
|
|
15
|
+
) -> None:
|
|
16
|
+
"""Report whether a key is loaded. Never prints the key. Agents must not read TYPESAFE_* vars or key files."""
|
|
17
|
+
try:
|
|
18
|
+
config = load_config(key=key, creds=creds, model=model)
|
|
19
|
+
except (OSError, ValueError) as exc:
|
|
20
|
+
fail(code="usage", message=str(exc), exit_code=2)
|
|
21
|
+
emit_success(
|
|
22
|
+
data={
|
|
23
|
+
"has_key": bool(config.api_key),
|
|
24
|
+
"base_url": config.base_url,
|
|
25
|
+
"model": config.model,
|
|
26
|
+
},
|
|
27
|
+
command="auth status",
|
|
28
|
+
)
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import json
|
|
4
|
+
from pathlib import Path
|
|
5
|
+
|
|
6
|
+
import typer
|
|
7
|
+
|
|
8
|
+
from typesafe_cli.io import emit_success, fail
|
|
9
|
+
from typesafe_cli.questions import QuestionError
|
|
10
|
+
from typesafe_cli.recipes.decide import apply_decide, extract_answers, parse_noul_band
|
|
11
|
+
from typesafe_cli.recipes.thresholds import CHOICE_MIN_TOP_P, NOUL_UNCERTAIN_HIGH, NOUL_UNCERTAIN_LOW
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def decide(
|
|
15
|
+
answers_file: Path = typer.Option(..., "--answers-file", help="JSON envelope or answers object"),
|
|
16
|
+
noul_band: str = typer.Option(
|
|
17
|
+
f"{NOUL_UNCERTAIN_LOW}:{NOUL_UNCERTAIN_HIGH}",
|
|
18
|
+
"--noul-band",
|
|
19
|
+
help="Inclusive uncertain band low:high",
|
|
20
|
+
),
|
|
21
|
+
choice_min_p: float = typer.Option(
|
|
22
|
+
CHOICE_MIN_TOP_P,
|
|
23
|
+
"--choice-min-p",
|
|
24
|
+
help="Abstain when the winning choice probability is below this",
|
|
25
|
+
),
|
|
26
|
+
) -> None:
|
|
27
|
+
"""Map raw Jev answers to no/uncertain/yes (or abstain) without another HTTP call."""
|
|
28
|
+
try:
|
|
29
|
+
payload = json.loads(answers_file.read_text(encoding="utf-8"))
|
|
30
|
+
answers = extract_answers(payload)
|
|
31
|
+
low, high = parse_noul_band(noul_band)
|
|
32
|
+
if not 0.0 <= choice_min_p <= 1.0:
|
|
33
|
+
raise QuestionError("--choice-min-p must be between 0 and 1")
|
|
34
|
+
decisions = apply_decide(answers, noul_low=low, noul_high=high, choice_min_p=choice_min_p)
|
|
35
|
+
except (OSError, json.JSONDecodeError, QuestionError) as exc:
|
|
36
|
+
fail(code="usage", message=str(exc), exit_code=2)
|
|
37
|
+
emit_success(
|
|
38
|
+
data={
|
|
39
|
+
"decisions": decisions,
|
|
40
|
+
"policy": {"noul_band": [low, high], "choice_min_p": choice_min_p},
|
|
41
|
+
},
|
|
42
|
+
command="decide",
|
|
43
|
+
)
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from pathlib import Path
|
|
4
|
+
from typing import Any
|
|
5
|
+
|
|
6
|
+
from typesafe_sdk import TypeSafeError
|
|
7
|
+
|
|
8
|
+
from typesafe_cli.client import system_one
|
|
9
|
+
from typesafe_cli.config import load_config
|
|
10
|
+
from typesafe_cli.io import fail
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def ready_config(*, key: str | None, creds: Path | None, model: str | None):
|
|
14
|
+
try:
|
|
15
|
+
config = load_config(key=key, creds=creds, model=model)
|
|
16
|
+
except (OSError, ValueError) as exc:
|
|
17
|
+
fail(code="usage", message=str(exc), exit_code=2)
|
|
18
|
+
if not config.api_key:
|
|
19
|
+
fail(code="auth", message="missing TYPESAFE_API_KEY (or --key / --creds)", exit_code=1)
|
|
20
|
+
return config
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def call_system_one(
|
|
24
|
+
*,
|
|
25
|
+
state: str | dict | list,
|
|
26
|
+
questions: dict[str, dict[str, Any]],
|
|
27
|
+
key: str | None,
|
|
28
|
+
creds: Path | None,
|
|
29
|
+
model: str | None,
|
|
30
|
+
) -> dict[str, Any]:
|
|
31
|
+
config = ready_config(key=key, creds=creds, model=model)
|
|
32
|
+
try:
|
|
33
|
+
return system_one(config=config, state=state, questions=questions, model=model)
|
|
34
|
+
except TypeSafeError as exc:
|
|
35
|
+
fail(code="request", message=str(exc), exit_code=1)
|
|
36
|
+
except Exception as exc: # noqa: BLE001
|
|
37
|
+
fail(code="request", message=str(exc), exit_code=1)
|
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import json
|
|
4
|
+
from pathlib import Path
|
|
5
|
+
|
|
6
|
+
import typer
|
|
7
|
+
|
|
8
|
+
from typesafe_cli.commands.eval import call_system_one
|
|
9
|
+
from typesafe_cli.io import emit_success, fail
|
|
10
|
+
from typesafe_cli.questions import QuestionError
|
|
11
|
+
from typesafe_cli.recipes.extract import extract_questions, find_spans
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def extract(
|
|
15
|
+
question: str = typer.Option(..., "--question", help="Which span to pick"),
|
|
16
|
+
file: Path | None = typer.Option(None, "--file", help="Document to scan"),
|
|
17
|
+
state_file: Path | None = typer.Option(None, "--state-file", help="JSON or text document"),
|
|
18
|
+
pattern: str | None = typer.Option(None, "--pattern", help="email, phone, or money"),
|
|
19
|
+
candidates: Path | None = typer.Option(None, "--candidates", help="JSON array of candidate strings"),
|
|
20
|
+
model: str | None = typer.Option(None, "--model"),
|
|
21
|
+
key: str | None = typer.Option(None, "--key"),
|
|
22
|
+
creds: Path | None = typer.Option(None, "--creds"),
|
|
23
|
+
) -> None:
|
|
24
|
+
"""Pick a verbatim value from regex (or supplied) spans. Jev does not invent the string."""
|
|
25
|
+
try:
|
|
26
|
+
text = _document_text(file=file, state_file=state_file)
|
|
27
|
+
spans = _spans(text=text, pattern=pattern, candidates=candidates)
|
|
28
|
+
except (OSError, json.JSONDecodeError, QuestionError) as exc:
|
|
29
|
+
fail(code="usage", message=str(exc), exit_code=2)
|
|
30
|
+
|
|
31
|
+
if not spans:
|
|
32
|
+
emit_success(
|
|
33
|
+
data={"value": None, "choice": "none", "skipped": True, "candidates": []},
|
|
34
|
+
command="extract",
|
|
35
|
+
)
|
|
36
|
+
return
|
|
37
|
+
|
|
38
|
+
data = call_system_one(
|
|
39
|
+
state={"document": text, "candidates": spans, "question": question},
|
|
40
|
+
questions=extract_questions(spans, question),
|
|
41
|
+
key=key,
|
|
42
|
+
creds=creds,
|
|
43
|
+
model=model,
|
|
44
|
+
)
|
|
45
|
+
answer = data["answers"]["value"]
|
|
46
|
+
choice = answer["choice"]
|
|
47
|
+
value = None if choice == "none" else choice
|
|
48
|
+
emit_success(
|
|
49
|
+
data={
|
|
50
|
+
"value": value,
|
|
51
|
+
"choice": choice,
|
|
52
|
+
"confidence": answer.get("confidence"),
|
|
53
|
+
"candidates": spans,
|
|
54
|
+
"skipped": False,
|
|
55
|
+
"model": data.get("model"),
|
|
56
|
+
"usage": data.get("usage"),
|
|
57
|
+
},
|
|
58
|
+
command="extract",
|
|
59
|
+
)
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def _document_text(*, file: Path | None, state_file: Path | None) -> str:
|
|
63
|
+
provided = int(file is not None) + int(state_file is not None)
|
|
64
|
+
if provided != 1:
|
|
65
|
+
raise QuestionError("exactly one of --file or --state-file is required")
|
|
66
|
+
path = file or state_file
|
|
67
|
+
assert path is not None
|
|
68
|
+
return path.read_text(encoding="utf-8")
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def _spans(*, text: str, pattern: str | None, candidates: Path | None) -> list[str]:
|
|
72
|
+
if candidates is not None:
|
|
73
|
+
raw = json.loads(candidates.read_text(encoding="utf-8"))
|
|
74
|
+
if not isinstance(raw, list) or not all(isinstance(item, str) for item in raw):
|
|
75
|
+
raise QuestionError("--candidates must be a JSON array of strings")
|
|
76
|
+
seen: list[str] = []
|
|
77
|
+
for item in raw:
|
|
78
|
+
if item not in seen:
|
|
79
|
+
seen.append(item)
|
|
80
|
+
return seen
|
|
81
|
+
if pattern is None:
|
|
82
|
+
raise QuestionError("--pattern or --candidates is required")
|
|
83
|
+
return find_spans(text, pattern)
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from pathlib import Path
|
|
4
|
+
|
|
5
|
+
import typer
|
|
6
|
+
|
|
7
|
+
from typesafe_cli.commands.eval import call_system_one
|
|
8
|
+
from typesafe_cli.io import emit_success, fail
|
|
9
|
+
from typesafe_cli.questions import QuestionError
|
|
10
|
+
from typesafe_cli.recipes.find import FIND_CHUNK, find_questions, merge_find_results, tag_lines, tagged_state, windows
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def find(
|
|
14
|
+
file: Path = typer.Option(..., "--file", help="Local file to search"),
|
|
15
|
+
query: str = typer.Option(..., "--query", help="Plain-language question"),
|
|
16
|
+
model: str | None = typer.Option(None, "--model"),
|
|
17
|
+
key: str | None = typer.Option(None, "--key"),
|
|
18
|
+
creds: Path | None = typer.Option(None, "--creds"),
|
|
19
|
+
) -> None:
|
|
20
|
+
"""Semantic line search: tag lines, Choice over ids, Noul whether an answer exists."""
|
|
21
|
+
try:
|
|
22
|
+
text = file.read_text(encoding="utf-8")
|
|
23
|
+
lines = tag_lines(text)
|
|
24
|
+
if not lines:
|
|
25
|
+
raise QuestionError("file is empty")
|
|
26
|
+
except (OSError, QuestionError) as exc:
|
|
27
|
+
fail(code="usage", message=str(exc), exit_code=2)
|
|
28
|
+
|
|
29
|
+
by_id = {line["id"]: line["text"] for line in lines}
|
|
30
|
+
chunks = []
|
|
31
|
+
for window in windows(lines, size=FIND_CHUNK):
|
|
32
|
+
data = call_system_one(
|
|
33
|
+
state=tagged_state(window),
|
|
34
|
+
questions=find_questions(window, query),
|
|
35
|
+
key=key,
|
|
36
|
+
creds=creds,
|
|
37
|
+
model=model,
|
|
38
|
+
)
|
|
39
|
+
chunks.append(data)
|
|
40
|
+
|
|
41
|
+
merged = merge_find_results(lines_by_id=by_id, chunks=chunks)
|
|
42
|
+
emit_success(
|
|
43
|
+
data={
|
|
44
|
+
"query": query,
|
|
45
|
+
"file": str(file),
|
|
46
|
+
"verdict": merged["verdict"],
|
|
47
|
+
"exists": merged["exists"],
|
|
48
|
+
"lines": merged["query_lines"][:20],
|
|
49
|
+
"model": merged["model"],
|
|
50
|
+
"usage": merged["usage"],
|
|
51
|
+
},
|
|
52
|
+
command="find",
|
|
53
|
+
)
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from pathlib import Path
|
|
4
|
+
|
|
5
|
+
import typer
|
|
6
|
+
from typesafe_sdk import TypeSafeError
|
|
7
|
+
|
|
8
|
+
from typesafe_cli.client import list_models
|
|
9
|
+
from typesafe_cli.config import load_config
|
|
10
|
+
from typesafe_cli.io import emit_success, fail
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def models(
|
|
14
|
+
key: str | None = typer.Option(None, "--key"),
|
|
15
|
+
creds: Path | None = typer.Option(None, "--creds"),
|
|
16
|
+
model: str | None = typer.Option(None, "--model"),
|
|
17
|
+
) -> None:
|
|
18
|
+
try:
|
|
19
|
+
config = load_config(key=key, creds=creds, model=model)
|
|
20
|
+
except (OSError, ValueError) as exc:
|
|
21
|
+
fail(code="usage", message=str(exc), exit_code=2)
|
|
22
|
+
if not config.api_key:
|
|
23
|
+
fail(code="auth", message="missing TYPESAFE_API_KEY (or --key / --creds)", exit_code=1)
|
|
24
|
+
try:
|
|
25
|
+
data = list_models(config=config)
|
|
26
|
+
except TypeSafeError as exc:
|
|
27
|
+
fail(code="request", message=str(exc), exit_code=1)
|
|
28
|
+
except Exception as exc: # noqa: BLE001
|
|
29
|
+
fail(code="request", message=str(exc), exit_code=1)
|
|
30
|
+
emit_success(data=data, command="models")
|
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import sys
|
|
4
|
+
from pathlib import Path
|
|
5
|
+
|
|
6
|
+
import typer
|
|
7
|
+
|
|
8
|
+
from typesafe_cli.commands.ask import run_evaluation
|
|
9
|
+
from typesafe_cli.io import fail
|
|
10
|
+
from typesafe_cli.questions import QuestionError, load_questions, load_state_file, parse_state_text
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def _state_from_flags(state: str | None, state_file: Path | None) -> str | dict | list:
|
|
14
|
+
provided = int(state is not None) + int(state_file is not None)
|
|
15
|
+
if provided != 1:
|
|
16
|
+
raise QuestionError("exactly one of --state or --state-file is required")
|
|
17
|
+
if state is not None:
|
|
18
|
+
return parse_state_text(state)
|
|
19
|
+
assert state_file is not None
|
|
20
|
+
if str(state_file) == "-":
|
|
21
|
+
return parse_state_text(sys.stdin.read())
|
|
22
|
+
return load_state_file(state_file)
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def noul(
|
|
26
|
+
instructions: str = typer.Argument(..., help="Yes/no question"),
|
|
27
|
+
state: str | None = typer.Option(None, "--state"),
|
|
28
|
+
state_file: Path | None = typer.Option(None, "--state-file"),
|
|
29
|
+
question_id: str = typer.Option("noul", "--id"),
|
|
30
|
+
model: str | None = typer.Option(None, "--model"),
|
|
31
|
+
key: str | None = typer.Option(None, "--key"),
|
|
32
|
+
creds: Path | None = typer.Option(None, "--creds"),
|
|
33
|
+
) -> None:
|
|
34
|
+
try:
|
|
35
|
+
resolved = _state_from_flags(state, state_file)
|
|
36
|
+
questions = load_questions({question_id: {"type": "noul", "instructions": instructions}})
|
|
37
|
+
except QuestionError as exc:
|
|
38
|
+
fail(code="usage", message=str(exc), exit_code=2)
|
|
39
|
+
run_evaluation(state=resolved, questions=questions, key=key, creds=creds, model=model, command="noul")
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def choice(
|
|
43
|
+
instructions: str = typer.Argument(..., help="Choice question"),
|
|
44
|
+
state: str | None = typer.Option(None, "--state"),
|
|
45
|
+
state_file: Path | None = typer.Option(None, "--state-file"),
|
|
46
|
+
option: list[str] = typer.Option(..., "--option", help="Choice option; at least two"),
|
|
47
|
+
question_id: str = typer.Option("choice", "--id"),
|
|
48
|
+
model: str | None = typer.Option(None, "--model"),
|
|
49
|
+
key: str | None = typer.Option(None, "--key"),
|
|
50
|
+
creds: Path | None = typer.Option(None, "--creds"),
|
|
51
|
+
) -> None:
|
|
52
|
+
criteria = _parse_named_flags(option)
|
|
53
|
+
try:
|
|
54
|
+
resolved = _state_from_flags(state, state_file)
|
|
55
|
+
questions = load_questions(
|
|
56
|
+
{question_id: {"type": "choice", "instructions": instructions, "criteria": criteria}}
|
|
57
|
+
)
|
|
58
|
+
except QuestionError as exc:
|
|
59
|
+
fail(code="usage", message=str(exc), exit_code=2)
|
|
60
|
+
run_evaluation(state=resolved, questions=questions, key=key, creds=creds, model=model, command="choice")
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def score(
|
|
64
|
+
instructions: str = typer.Argument(..., help="Score question"),
|
|
65
|
+
state: str | None = typer.Option(None, "--state"),
|
|
66
|
+
state_file: Path | None = typer.Option(None, "--state-file"),
|
|
67
|
+
level: list[str] = typer.Option(..., "--level", help="Score level, lowest first; at least two"),
|
|
68
|
+
question_id: str = typer.Option("score", "--id"),
|
|
69
|
+
model: str | None = typer.Option(None, "--model"),
|
|
70
|
+
key: str | None = typer.Option(None, "--key"),
|
|
71
|
+
creds: Path | None = typer.Option(None, "--creds"),
|
|
72
|
+
) -> None:
|
|
73
|
+
try:
|
|
74
|
+
resolved = _state_from_flags(state, state_file)
|
|
75
|
+
questions = load_questions(
|
|
76
|
+
{question_id: {"type": "score", "instructions": instructions, "criteria": level}}
|
|
77
|
+
)
|
|
78
|
+
except QuestionError as exc:
|
|
79
|
+
fail(code="usage", message=str(exc), exit_code=2)
|
|
80
|
+
run_evaluation(state=resolved, questions=questions, key=key, creds=creds, model=model, command="score")
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
def _parse_named_flags(values: list[str]) -> dict[str, str | None]:
|
|
84
|
+
criteria: dict[str, str | None] = {}
|
|
85
|
+
for raw in values:
|
|
86
|
+
if "=" in raw:
|
|
87
|
+
name, description = raw.split("=", 1)
|
|
88
|
+
criteria[name] = description
|
|
89
|
+
else:
|
|
90
|
+
criteria[raw] = None
|
|
91
|
+
return criteria
|