flintai-cli 1.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- flintai/__init__.py +0 -0
- flintai/cli/__init__.py +0 -0
- flintai/cli/__main__.py +3 -0
- flintai/cli/builtin_config.json +911 -0
- flintai/cli/console.py +116 -0
- flintai/cli/eval_cli.py +891 -0
- flintai/cli/init_cli.py +144 -0
- flintai/cli/main.py +262 -0
- flintai/cli/rich_observer.py +89 -0
- flintai/cli/runner.py +305 -0
- flintai/cli/scan_cli.py +238 -0
- flintai/cli/utils.py +37 -0
- flintai/cli/version.py +1 -0
- flintai/data/detector_prompts/llm01_adversarial.txt +55 -0
- flintai/data/detector_prompts/llm01_fixed.txt +8 -0
- flintai/data/detector_prompts/llm02_adversarial.txt +75 -0
- flintai/data/detector_prompts/llm02_fixed.txt +8 -0
- flintai/data/detector_prompts/llm05_adversarial.txt +63 -0
- flintai/data/detector_prompts/llm05_fixed.txt +16 -0
- flintai/data/detector_prompts/llm06_adversarial.txt +135 -0
- flintai/data/detector_prompts/llm06_fixed.txt +15 -0
- flintai/data/detector_prompts/llm07_adversarial.txt +185 -0
- flintai/data/detector_prompts/llm07_fixed.txt +14 -0
- flintai/data/detector_prompts/llm09_adversarial.txt +77 -0
- flintai/data/detector_prompts/llm09_fixed.txt +15 -0
- flintai/data/llm01_prompt_injection.csv +6708 -0
- flintai/data/llm02_sensitive_information.csv +1069 -0
- flintai/data/llm05_unsafe_output.csv +3882 -0
- flintai/data/llm06_excessive_agency.csv +778 -0
- flintai/data/llm07_system_prompt_leakage.csv +971 -0
- flintai/data/llm09_hallucination-goals.csv +32419 -0
- flintai/data/llm09_hallucination.csv +599 -0
- flintai/data/pii_leakage.csv +706 -0
- flintai/data/secret_leakage.csv +1380 -0
- flintai/eval/__init__.py +0 -0
- flintai/eval/common/converter_anthropic.py +151 -0
- flintai/eval/common/converter_genai.py +169 -0
- flintai/eval/common/converter_openai.py +165 -0
- flintai/eval/common/log.py +96 -0
- flintai/eval/common/reference.py +25 -0
- flintai/eval/common/schema.py +192 -0
- flintai/eval/common/utils.py +74 -0
- flintai/eval/core/__init__.py +0 -0
- flintai/eval/core/detectors/__init__.py +1 -0
- flintai/eval/core/detectors/detector.py +25 -0
- flintai/eval/core/detectors/detector_garak.py +80 -0
- flintai/eval/core/detectors/detector_model.py +76 -0
- flintai/eval/core/detectors/detector_pii.py +64 -0
- flintai/eval/core/detectors/detector_secret.py +82 -0
- flintai/eval/core/detectors/detector_topic_guard.py +70 -0
- flintai/eval/core/detectors/detector_toxicity.py +70 -0
- flintai/eval/core/eval/__init__.py +0 -0
- flintai/eval/core/eval/eval_creator.py +197 -0
- flintai/eval/core/eval/evaluation.py +100 -0
- flintai/eval/core/eval/evaluation_adversarial.py +523 -0
- flintai/eval/core/eval/evaluation_garak_module.py +52 -0
- flintai/eval/core/eval/evaluation_garak_probe.py +274 -0
- flintai/eval/core/eval/evaluation_message_list.py +44 -0
- flintai/eval/core/eval/evaluation_multi.py +207 -0
- flintai/eval/core/eval/evaluation_single.py +79 -0
- flintai/eval/core/eval/evaluation_single_prompt.py +58 -0
- flintai/eval/core/eval/evaluation_topic_guard.py +318 -0
- flintai/eval/core/eval/metric_conciseness.py +181 -0
- flintai/eval/core/eval/metric_factual_accuracy.py +294 -0
- flintai/eval/core/eval/metric_instruction_adherence.py +180 -0
- flintai/eval/core/eval/metric_tone.py +183 -0
- flintai/eval/core/eval/metric_toxicity.py +147 -0
- flintai/eval/core/eval/observer.py +136 -0
- flintai/eval/core/eval/probe_adversarial.py +14 -0
- flintai/eval/core/message/__init__.py +0 -0
- flintai/eval/core/message/message_collection.py +37 -0
- flintai/eval/core/message/message_collection_csv.py +46 -0
- flintai/eval/core/message/message_collection_garak.py +52 -0
- flintai/eval/core/message/message_collection_memory.py +36 -0
- flintai/eval/core/models/__init__.py +0 -0
- flintai/eval/core/models/generator_model.py +88 -0
- flintai/eval/core/models/model.py +111 -0
- flintai/eval/core/models/model_adk.py +158 -0
- flintai/eval/core/models/model_anthropic.py +51 -0
- flintai/eval/core/models/model_anthropic_agent.py +92 -0
- flintai/eval/core/models/model_gemini.py +107 -0
- flintai/eval/core/models/model_generic_http.py +117 -0
- flintai/eval/core/models/model_huggingface.py +61 -0
- flintai/eval/core/models/model_langserve.py +94 -0
- flintai/eval/core/models/model_litellm.py +38 -0
- flintai/eval/core/models/model_ollama.py +50 -0
- flintai/eval/core/models/model_openai.py +35 -0
- flintai/eval/core/models/model_openai_agent.py +80 -0
- flintai/eval/core/models/model_openai_compatible.py +57 -0
- flintai/eval/core/models/model_retry.py +134 -0
- flintai/eval/core/models/model_sync_wrapper.py +35 -0
- flintai/eval/db/__init__.py +0 -0
- flintai/eval/db/base/__init__.py +0 -0
- flintai/eval/db/base/detectors/__init__.py +1 -0
- flintai/eval/db/base/detectors/detector_helpers.py +78 -0
- flintai/eval/db/base/detectors/detector_repository.py +91 -0
- flintai/eval/db/base/detectors/detector_types.py +77 -0
- flintai/eval/db/base/eval/__init__.py +1 -0
- flintai/eval/db/base/eval/eval_helpers.py +227 -0
- flintai/eval/db/base/eval/eval_repository.py +115 -0
- flintai/eval/db/base/eval/eval_run.py +139 -0
- flintai/eval/db/base/eval/eval_types.py +129 -0
- flintai/eval/db/base/eval/model_eval_repository.py +78 -0
- flintai/eval/db/base/eval/model_eval_run_repository.py +70 -0
- flintai/eval/db/base/eval/model_eval_run_result_repository.py +40 -0
- flintai/eval/db/base/eval/model_eval_run_result_types.py +21 -0
- flintai/eval/db/base/eval/model_eval_run_types.py +84 -0
- flintai/eval/db/base/eval/model_eval_types.py +71 -0
- flintai/eval/db/base/message/__init__.py +0 -0
- flintai/eval/db/base/message/message_collection_helpers.py +61 -0
- flintai/eval/db/base/message/message_collection_repository.py +92 -0
- flintai/eval/db/base/message/message_collection_types.py +89 -0
- flintai/eval/db/base/models/__init__.py +0 -0
- flintai/eval/db/base/models/model_helpers.py +172 -0
- flintai/eval/db/base/models/model_repository.py +95 -0
- flintai/eval/db/base/models/model_types.py +156 -0
- flintai/eval/db/json/__init__.py +0 -0
- flintai/eval/db/json/repository_json.py +603 -0
- flintai/scan/__init__.py +0 -0
- flintai/scan/agent_scanner.py +797 -0
- flintai/scan/config/agent_opengrep_rules.yaml +533 -0
- flintai/scan/config/agent_taxonomy.json +481 -0
- flintai/scan/config/agentic_cvss_mapping.yaml +136 -0
- flintai/scan/config/compliance_mappings.json +110 -0
- flintai/scan/config/triage_prompt.txt +335 -0
- flintai/scan/constants.py +15 -0
- flintai/scan/file_filter.py +156 -0
- flintai/scan/llm_provider.py +197 -0
- flintai/scan/opengrep_resolver.py +12 -0
- flintai/scan/reasoner.py +552 -0
- flintai/scan/schema.py +337 -0
- flintai/scan/scorer.py +371 -0
- flintai/scan/secret_anonymizer.py +72 -0
- flintai/scan/static_scanner.py +542 -0
- flintai/scan/taxonomy.py +75 -0
- flintai/scan/tool_dispatcher.py +754 -0
- flintai/scan/trace_logger.py +118 -0
- flintai/scan/trace_logger_file.py +143 -0
- flintai/scan/trace_logger_log.py +100 -0
- flintai/scan/triage.py +496 -0
- flintai/schema.py +58 -0
- flintai_cli-1.0.0.dist-info/METADATA +442 -0
- flintai_cli-1.0.0.dist-info/RECORD +147 -0
- flintai_cli-1.0.0.dist-info/WHEEL +5 -0
- flintai_cli-1.0.0.dist-info/entry_points.txt +2 -0
- flintai_cli-1.0.0.dist-info/licenses/LICENSE +210 -0
- flintai_cli-1.0.0.dist-info/top_level.txt +1 -0
flintai/cli/console.py
ADDED
|
@@ -0,0 +1,116 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import sys
|
|
4
|
+
import tty
|
|
5
|
+
import termios
|
|
6
|
+
|
|
7
|
+
from rich.console import Console
|
|
8
|
+
from rich.prompt import Prompt
|
|
9
|
+
from rich.theme import Theme
|
|
10
|
+
|
|
11
|
+
CLI_WIDTH = 120
|
|
12
|
+
|
|
13
|
+
_theme = Theme({
|
|
14
|
+
"score.high": "bold green",
|
|
15
|
+
"score.mid": "bold yellow",
|
|
16
|
+
"score.low": "bold red",
|
|
17
|
+
"status.finished": "bold green",
|
|
18
|
+
"status.running": "bold blue",
|
|
19
|
+
"status.error": "bold red",
|
|
20
|
+
"status.waiting": "dim",
|
|
21
|
+
"status.initializing": "bold cyan",
|
|
22
|
+
"status.initialized": "cyan",
|
|
23
|
+
"key": "dim",
|
|
24
|
+
})
|
|
25
|
+
|
|
26
|
+
console = Console(theme=_theme, width=CLI_WIDTH)
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def score_style(score: float) -> str:
|
|
30
|
+
if score >= 0.7:
|
|
31
|
+
return "score.high"
|
|
32
|
+
if score >= 0.4:
|
|
33
|
+
return "score.mid"
|
|
34
|
+
return "score.low"
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def status_style(status: str) -> str:
|
|
38
|
+
normalized = status.lower().strip()
|
|
39
|
+
return f"status.{normalized}" if normalized in (
|
|
40
|
+
"finished", "running", "error", "waiting",
|
|
41
|
+
"initializing", "initialized",
|
|
42
|
+
) else ""
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def severity_style(severity: str) -> str:
|
|
46
|
+
s = severity.lower().strip()
|
|
47
|
+
if s == "critical":
|
|
48
|
+
return "bold red"
|
|
49
|
+
if s == "high":
|
|
50
|
+
return "red"
|
|
51
|
+
if s == "medium":
|
|
52
|
+
return "yellow"
|
|
53
|
+
return "green"
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def select(
|
|
57
|
+
prompt: str,
|
|
58
|
+
options: list[str],
|
|
59
|
+
*,
|
|
60
|
+
default_index: int = 0,
|
|
61
|
+
) -> str:
|
|
62
|
+
if not sys.stdin.isatty():
|
|
63
|
+
return Prompt.ask(prompt, choices=options, default=options[default_index])
|
|
64
|
+
|
|
65
|
+
if len(options) == 1:
|
|
66
|
+
return options[0]
|
|
67
|
+
|
|
68
|
+
index = default_index
|
|
69
|
+
fd = sys.stdin.fileno()
|
|
70
|
+
old_settings = termios.tcgetattr(fd)
|
|
71
|
+
|
|
72
|
+
def _render() -> None:
|
|
73
|
+
console.print(f"[bold cyan]{prompt}[/bold cyan]")
|
|
74
|
+
for i, option in enumerate(options):
|
|
75
|
+
if i == index:
|
|
76
|
+
console.print(f" [bold green]> {option}[/bold green]")
|
|
77
|
+
else:
|
|
78
|
+
console.print(f" [dim]{option}[/dim]")
|
|
79
|
+
|
|
80
|
+
def _clear_lines(n: int) -> None:
|
|
81
|
+
for _ in range(n):
|
|
82
|
+
sys.stdout.write("\033[A\033[2K")
|
|
83
|
+
sys.stdout.write("\r")
|
|
84
|
+
sys.stdout.flush()
|
|
85
|
+
|
|
86
|
+
try:
|
|
87
|
+
_render()
|
|
88
|
+
tty.setraw(fd)
|
|
89
|
+
|
|
90
|
+
while True:
|
|
91
|
+
ch = sys.stdin.read(1)
|
|
92
|
+
|
|
93
|
+
if ch == "\r" or ch == "\n":
|
|
94
|
+
break
|
|
95
|
+
|
|
96
|
+
if ch == "\x03":
|
|
97
|
+
raise KeyboardInterrupt
|
|
98
|
+
|
|
99
|
+
if ch == "\x1b":
|
|
100
|
+
seq = sys.stdin.read(2)
|
|
101
|
+
if seq == "[A":
|
|
102
|
+
index = (index - 1) % len(options)
|
|
103
|
+
elif seq == "[B":
|
|
104
|
+
index = (index + 1) % len(options)
|
|
105
|
+
|
|
106
|
+
termios.tcsetattr(fd, termios.TCSADRAIN, old_settings)
|
|
107
|
+
_clear_lines(len(options) + 1)
|
|
108
|
+
_render()
|
|
109
|
+
tty.setraw(fd)
|
|
110
|
+
finally:
|
|
111
|
+
termios.tcsetattr(fd, termios.TCSADRAIN, old_settings)
|
|
112
|
+
|
|
113
|
+
_clear_lines(len(options) + 1)
|
|
114
|
+
console.print(f"[bold cyan]{prompt}[/bold cyan]: [green]{options[index]}[/green]")
|
|
115
|
+
|
|
116
|
+
return options[index]
|