flintai-cli 1.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- flintai/__init__.py +0 -0
- flintai/cli/__init__.py +0 -0
- flintai/cli/__main__.py +3 -0
- flintai/cli/builtin_config.json +911 -0
- flintai/cli/console.py +116 -0
- flintai/cli/eval_cli.py +891 -0
- flintai/cli/init_cli.py +144 -0
- flintai/cli/main.py +262 -0
- flintai/cli/rich_observer.py +89 -0
- flintai/cli/runner.py +305 -0
- flintai/cli/scan_cli.py +238 -0
- flintai/cli/utils.py +37 -0
- flintai/cli/version.py +1 -0
- flintai/data/detector_prompts/llm01_adversarial.txt +55 -0
- flintai/data/detector_prompts/llm01_fixed.txt +8 -0
- flintai/data/detector_prompts/llm02_adversarial.txt +75 -0
- flintai/data/detector_prompts/llm02_fixed.txt +8 -0
- flintai/data/detector_prompts/llm05_adversarial.txt +63 -0
- flintai/data/detector_prompts/llm05_fixed.txt +16 -0
- flintai/data/detector_prompts/llm06_adversarial.txt +135 -0
- flintai/data/detector_prompts/llm06_fixed.txt +15 -0
- flintai/data/detector_prompts/llm07_adversarial.txt +185 -0
- flintai/data/detector_prompts/llm07_fixed.txt +14 -0
- flintai/data/detector_prompts/llm09_adversarial.txt +77 -0
- flintai/data/detector_prompts/llm09_fixed.txt +15 -0
- flintai/data/llm01_prompt_injection.csv +6708 -0
- flintai/data/llm02_sensitive_information.csv +1069 -0
- flintai/data/llm05_unsafe_output.csv +3882 -0
- flintai/data/llm06_excessive_agency.csv +778 -0
- flintai/data/llm07_system_prompt_leakage.csv +971 -0
- flintai/data/llm09_hallucination-goals.csv +32419 -0
- flintai/data/llm09_hallucination.csv +599 -0
- flintai/data/pii_leakage.csv +706 -0
- flintai/data/secret_leakage.csv +1380 -0
- flintai/eval/__init__.py +0 -0
- flintai/eval/common/converter_anthropic.py +151 -0
- flintai/eval/common/converter_genai.py +169 -0
- flintai/eval/common/converter_openai.py +165 -0
- flintai/eval/common/log.py +96 -0
- flintai/eval/common/reference.py +25 -0
- flintai/eval/common/schema.py +192 -0
- flintai/eval/common/utils.py +74 -0
- flintai/eval/core/__init__.py +0 -0
- flintai/eval/core/detectors/__init__.py +1 -0
- flintai/eval/core/detectors/detector.py +25 -0
- flintai/eval/core/detectors/detector_garak.py +80 -0
- flintai/eval/core/detectors/detector_model.py +76 -0
- flintai/eval/core/detectors/detector_pii.py +64 -0
- flintai/eval/core/detectors/detector_secret.py +82 -0
- flintai/eval/core/detectors/detector_topic_guard.py +70 -0
- flintai/eval/core/detectors/detector_toxicity.py +70 -0
- flintai/eval/core/eval/__init__.py +0 -0
- flintai/eval/core/eval/eval_creator.py +197 -0
- flintai/eval/core/eval/evaluation.py +100 -0
- flintai/eval/core/eval/evaluation_adversarial.py +523 -0
- flintai/eval/core/eval/evaluation_garak_module.py +52 -0
- flintai/eval/core/eval/evaluation_garak_probe.py +274 -0
- flintai/eval/core/eval/evaluation_message_list.py +44 -0
- flintai/eval/core/eval/evaluation_multi.py +207 -0
- flintai/eval/core/eval/evaluation_single.py +79 -0
- flintai/eval/core/eval/evaluation_single_prompt.py +58 -0
- flintai/eval/core/eval/evaluation_topic_guard.py +318 -0
- flintai/eval/core/eval/metric_conciseness.py +181 -0
- flintai/eval/core/eval/metric_factual_accuracy.py +294 -0
- flintai/eval/core/eval/metric_instruction_adherence.py +180 -0
- flintai/eval/core/eval/metric_tone.py +183 -0
- flintai/eval/core/eval/metric_toxicity.py +147 -0
- flintai/eval/core/eval/observer.py +136 -0
- flintai/eval/core/eval/probe_adversarial.py +14 -0
- flintai/eval/core/message/__init__.py +0 -0
- flintai/eval/core/message/message_collection.py +37 -0
- flintai/eval/core/message/message_collection_csv.py +46 -0
- flintai/eval/core/message/message_collection_garak.py +52 -0
- flintai/eval/core/message/message_collection_memory.py +36 -0
- flintai/eval/core/models/__init__.py +0 -0
- flintai/eval/core/models/generator_model.py +88 -0
- flintai/eval/core/models/model.py +111 -0
- flintai/eval/core/models/model_adk.py +158 -0
- flintai/eval/core/models/model_anthropic.py +51 -0
- flintai/eval/core/models/model_anthropic_agent.py +92 -0
- flintai/eval/core/models/model_gemini.py +107 -0
- flintai/eval/core/models/model_generic_http.py +117 -0
- flintai/eval/core/models/model_huggingface.py +61 -0
- flintai/eval/core/models/model_langserve.py +94 -0
- flintai/eval/core/models/model_litellm.py +38 -0
- flintai/eval/core/models/model_ollama.py +50 -0
- flintai/eval/core/models/model_openai.py +35 -0
- flintai/eval/core/models/model_openai_agent.py +80 -0
- flintai/eval/core/models/model_openai_compatible.py +57 -0
- flintai/eval/core/models/model_retry.py +134 -0
- flintai/eval/core/models/model_sync_wrapper.py +35 -0
- flintai/eval/db/__init__.py +0 -0
- flintai/eval/db/base/__init__.py +0 -0
- flintai/eval/db/base/detectors/__init__.py +1 -0
- flintai/eval/db/base/detectors/detector_helpers.py +78 -0
- flintai/eval/db/base/detectors/detector_repository.py +91 -0
- flintai/eval/db/base/detectors/detector_types.py +77 -0
- flintai/eval/db/base/eval/__init__.py +1 -0
- flintai/eval/db/base/eval/eval_helpers.py +227 -0
- flintai/eval/db/base/eval/eval_repository.py +115 -0
- flintai/eval/db/base/eval/eval_run.py +139 -0
- flintai/eval/db/base/eval/eval_types.py +129 -0
- flintai/eval/db/base/eval/model_eval_repository.py +78 -0
- flintai/eval/db/base/eval/model_eval_run_repository.py +70 -0
- flintai/eval/db/base/eval/model_eval_run_result_repository.py +40 -0
- flintai/eval/db/base/eval/model_eval_run_result_types.py +21 -0
- flintai/eval/db/base/eval/model_eval_run_types.py +84 -0
- flintai/eval/db/base/eval/model_eval_types.py +71 -0
- flintai/eval/db/base/message/__init__.py +0 -0
- flintai/eval/db/base/message/message_collection_helpers.py +61 -0
- flintai/eval/db/base/message/message_collection_repository.py +92 -0
- flintai/eval/db/base/message/message_collection_types.py +89 -0
- flintai/eval/db/base/models/__init__.py +0 -0
- flintai/eval/db/base/models/model_helpers.py +172 -0
- flintai/eval/db/base/models/model_repository.py +95 -0
- flintai/eval/db/base/models/model_types.py +156 -0
- flintai/eval/db/json/__init__.py +0 -0
- flintai/eval/db/json/repository_json.py +603 -0
- flintai/scan/__init__.py +0 -0
- flintai/scan/agent_scanner.py +797 -0
- flintai/scan/config/agent_opengrep_rules.yaml +533 -0
- flintai/scan/config/agent_taxonomy.json +481 -0
- flintai/scan/config/agentic_cvss_mapping.yaml +136 -0
- flintai/scan/config/compliance_mappings.json +110 -0
- flintai/scan/config/triage_prompt.txt +335 -0
- flintai/scan/constants.py +15 -0
- flintai/scan/file_filter.py +156 -0
- flintai/scan/llm_provider.py +197 -0
- flintai/scan/opengrep_resolver.py +12 -0
- flintai/scan/reasoner.py +552 -0
- flintai/scan/schema.py +337 -0
- flintai/scan/scorer.py +371 -0
- flintai/scan/secret_anonymizer.py +72 -0
- flintai/scan/static_scanner.py +542 -0
- flintai/scan/taxonomy.py +75 -0
- flintai/scan/tool_dispatcher.py +754 -0
- flintai/scan/trace_logger.py +118 -0
- flintai/scan/trace_logger_file.py +143 -0
- flintai/scan/trace_logger_log.py +100 -0
- flintai/scan/triage.py +496 -0
- flintai/schema.py +58 -0
- flintai_cli-1.0.0.dist-info/METADATA +442 -0
- flintai_cli-1.0.0.dist-info/RECORD +147 -0
- flintai_cli-1.0.0.dist-info/WHEEL +5 -0
- flintai_cli-1.0.0.dist-info/entry_points.txt +2 -0
- flintai_cli-1.0.0.dist-info/licenses/LICENSE +210 -0
- flintai_cli-1.0.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,197 @@
|
|
|
1
|
+
"""
|
|
2
|
+
llm_provider.py — LLM model factory for the agent scanner.
|
|
3
|
+
|
|
4
|
+
Single entry point:
|
|
5
|
+
|
|
6
|
+
make_model(model_string)
|
|
7
|
+
Returns an ADK-compatible LiteLlm model object for all providers.
|
|
8
|
+
Used by both the agentic reasoner (ADK Runner) and single-pass
|
|
9
|
+
completions (triage) via complete_text().
|
|
10
|
+
|
|
11
|
+
complete_text(model, system_prompt, user_message, ...)
|
|
12
|
+
Runs a single-pass LLM completion using generate_content_async
|
|
13
|
+
on the model returned by make_model().
|
|
14
|
+
|
|
15
|
+
Configuration via AGENT_SCANNER_MODEL env var in 'provider:model' format
|
|
16
|
+
(e.g., 'google:gemini-2.5-flash', 'openai:gpt-5.4', 'anthropic:claude-sonnet-4-6').
|
|
17
|
+
|
|
18
|
+
API keys are read from standard env vars by the underlying frameworks:
|
|
19
|
+
GOOGLE_API_KEY, OPENAI_API_KEY, ANTHROPIC_API_KEY.
|
|
20
|
+
"""
|
|
21
|
+
|
|
22
|
+
from __future__ import annotations
|
|
23
|
+
|
|
24
|
+
import asyncio
|
|
25
|
+
import concurrent.futures
|
|
26
|
+
import logging
|
|
27
|
+
import os
|
|
28
|
+
import re
|
|
29
|
+
import warnings
|
|
30
|
+
from typing import Any
|
|
31
|
+
|
|
32
|
+
from flintai.scan.schema import ADKModel
|
|
33
|
+
|
|
34
|
+
logger = logging.getLogger(__name__)
|
|
35
|
+
|
|
36
|
+
_REDACT_PATTERNS = re.compile(
|
|
37
|
+
r"("
|
|
38
|
+
r"Bearer\s+[A-Za-z0-9\-_.~+/]+=*"
|
|
39
|
+
r"|sk-[A-Za-z0-9]{10,}"
|
|
40
|
+
r"|key-[A-Za-z0-9]{10,}"
|
|
41
|
+
r"|pat-[A-Za-z0-9]{10,}"
|
|
42
|
+
r"|AIza[A-Za-z0-9_\-]{35,}"
|
|
43
|
+
r"|[A-Za-z0-9]{40,}"
|
|
44
|
+
r")",
|
|
45
|
+
re.IGNORECASE,
|
|
46
|
+
)
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def _safe_error(exc: Exception) -> str:
|
|
50
|
+
"""Return a log-safe string with token-like patterns redacted."""
|
|
51
|
+
return _REDACT_PATTERNS.sub("[REDACTED]", str(exc))
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
PROVIDER_GOOGLE = "google"
|
|
55
|
+
PROVIDER_LITELLM = "litellm"
|
|
56
|
+
|
|
57
|
+
DEFAULT_MODEL = "gemini-2.5-flash"
|
|
58
|
+
|
|
59
|
+
_PROVIDER_ALIASES = {"gemini": "google"}
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def parse_model_string(model_string: str) -> tuple[str, str | None]:
|
|
63
|
+
"""Parse 'provider:model' -> (provider, model)."""
|
|
64
|
+
if ":" in model_string:
|
|
65
|
+
provider, model = model_string.split(":", 1)
|
|
66
|
+
else:
|
|
67
|
+
provider, model = model_string, None
|
|
68
|
+
provider = provider.strip().lower()
|
|
69
|
+
provider = _PROVIDER_ALIASES.get(provider, provider)
|
|
70
|
+
return provider, model.strip() if model else None
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def _resolve_model_string(model_string: str | None = None) -> tuple[str, str]:
|
|
74
|
+
"""Resolve provider and model from argument or AGENT_SCANNER_MODEL env var."""
|
|
75
|
+
ms = (model_string or os.getenv("AGENT_SCANNER_MODEL", "")).strip()
|
|
76
|
+
if not ms:
|
|
77
|
+
return PROVIDER_GOOGLE, DEFAULT_MODEL
|
|
78
|
+
provider, model = parse_model_string(ms)
|
|
79
|
+
return provider, model or DEFAULT_MODEL
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def make_model(model_string: str | None = None, temperature: float = 0.0) -> ADKModel:
|
|
83
|
+
"""Return an ADK-compatible LiteLlm model for any provider.
|
|
84
|
+
|
|
85
|
+
Always returns a LiteLlm object so that all callers (agentic
|
|
86
|
+
reasoner, triage) use the same generate_content_async interface.
|
|
87
|
+
"""
|
|
88
|
+
provider, model = _resolve_model_string(model_string)
|
|
89
|
+
|
|
90
|
+
LiteLlm = _import_litellm()
|
|
91
|
+
if provider == PROVIDER_GOOGLE:
|
|
92
|
+
with warnings.catch_warnings(record=True) as caught:
|
|
93
|
+
warnings.simplefilter("always")
|
|
94
|
+
result = LiteLlm(model=f"gemini/{model}", temperature=temperature)
|
|
95
|
+
for w in caught:
|
|
96
|
+
logger.debug("%s", w.message)
|
|
97
|
+
return result
|
|
98
|
+
if provider == PROVIDER_LITELLM:
|
|
99
|
+
return LiteLlm(model=model, temperature=temperature)
|
|
100
|
+
return LiteLlm(model=f"{provider}/{model}", temperature=temperature)
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
def get_model_name(model_string: str | None = None) -> str:
|
|
104
|
+
"""Return the resolved model name for logging/metadata."""
|
|
105
|
+
_, model = _resolve_model_string(model_string)
|
|
106
|
+
return model
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
def _import_litellm() -> type[Any]:
|
|
110
|
+
"""Import and return the LiteLlm class from google-adk or a minimal shim."""
|
|
111
|
+
try:
|
|
112
|
+
from google.adk.models.lite_llm import LiteLlm
|
|
113
|
+
|
|
114
|
+
return LiteLlm
|
|
115
|
+
except ImportError:
|
|
116
|
+
pass
|
|
117
|
+
try:
|
|
118
|
+
from google.adk.models import LiteLlm # type: ignore[no-redef]
|
|
119
|
+
|
|
120
|
+
return LiteLlm
|
|
121
|
+
except ImportError:
|
|
122
|
+
pass
|
|
123
|
+
try:
|
|
124
|
+
import litellm as _litellm # noqa: F401
|
|
125
|
+
|
|
126
|
+
class _LiteLlmShim:
|
|
127
|
+
def __init__(self, model: str, **kwargs):
|
|
128
|
+
self.model = model
|
|
129
|
+
self._kwargs = kwargs
|
|
130
|
+
|
|
131
|
+
def __repr__(self) -> str:
|
|
132
|
+
return f"LiteLlm(model={self.model!r})"
|
|
133
|
+
|
|
134
|
+
return _LiteLlmShim
|
|
135
|
+
except ImportError as e:
|
|
136
|
+
raise ImportError(
|
|
137
|
+
"google-adk or litellm is required for non-Google providers. "
|
|
138
|
+
"Run: pip install google-adk litellm"
|
|
139
|
+
) from e
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
# ── Single-pass completion ───────────────────────────────────────────────────
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
def complete_text(
|
|
146
|
+
model: ADKModel,
|
|
147
|
+
system_prompt: str,
|
|
148
|
+
user_message: str,
|
|
149
|
+
max_tokens: int = 8000,
|
|
150
|
+
temperature: float = 0.0,
|
|
151
|
+
top_p: float = 1.0,
|
|
152
|
+
) -> str | None:
|
|
153
|
+
"""Run a single-pass LLM completion via model.generate_content_async."""
|
|
154
|
+
from google.adk.models.llm_request import LlmRequest # lazy: ADK optional
|
|
155
|
+
from google.genai import types as genai_types # lazy: ADK optional
|
|
156
|
+
|
|
157
|
+
model_name = getattr(model, "model", str(model))
|
|
158
|
+
|
|
159
|
+
config = genai_types.GenerateContentConfig(
|
|
160
|
+
system_instruction=system_prompt,
|
|
161
|
+
temperature=temperature,
|
|
162
|
+
top_p=top_p,
|
|
163
|
+
max_output_tokens=max_tokens,
|
|
164
|
+
)
|
|
165
|
+
contents = [
|
|
166
|
+
genai_types.Content(
|
|
167
|
+
role="user",
|
|
168
|
+
parts=[genai_types.Part.from_text(text=user_message)],
|
|
169
|
+
)
|
|
170
|
+
]
|
|
171
|
+
|
|
172
|
+
async def _run() -> str | None:
|
|
173
|
+
request = LlmRequest(
|
|
174
|
+
model=model_name,
|
|
175
|
+
contents=contents,
|
|
176
|
+
config=config,
|
|
177
|
+
)
|
|
178
|
+
text = ""
|
|
179
|
+
async for resp in model.generate_content_async(request):
|
|
180
|
+
if resp.content and resp.content.parts:
|
|
181
|
+
for part in resp.content.parts:
|
|
182
|
+
if hasattr(part, "text") and part.text:
|
|
183
|
+
text += part.text
|
|
184
|
+
return text.strip() or None
|
|
185
|
+
|
|
186
|
+
try:
|
|
187
|
+
asyncio.get_running_loop()
|
|
188
|
+
with concurrent.futures.ThreadPoolExecutor() as pool:
|
|
189
|
+
return pool.submit(asyncio.run, _run()).result(timeout=120)
|
|
190
|
+
except RuntimeError:
|
|
191
|
+
pass
|
|
192
|
+
|
|
193
|
+
try:
|
|
194
|
+
return asyncio.run(_run())
|
|
195
|
+
except Exception as e:
|
|
196
|
+
logger.error("LLM call failed: %s", _safe_error(e))
|
|
197
|
+
return None
|