flintai-cli 1.0.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (147) hide show
  1. flintai/__init__.py +0 -0
  2. flintai/cli/__init__.py +0 -0
  3. flintai/cli/__main__.py +3 -0
  4. flintai/cli/builtin_config.json +911 -0
  5. flintai/cli/console.py +116 -0
  6. flintai/cli/eval_cli.py +891 -0
  7. flintai/cli/init_cli.py +144 -0
  8. flintai/cli/main.py +262 -0
  9. flintai/cli/rich_observer.py +89 -0
  10. flintai/cli/runner.py +305 -0
  11. flintai/cli/scan_cli.py +238 -0
  12. flintai/cli/utils.py +37 -0
  13. flintai/cli/version.py +1 -0
  14. flintai/data/detector_prompts/llm01_adversarial.txt +55 -0
  15. flintai/data/detector_prompts/llm01_fixed.txt +8 -0
  16. flintai/data/detector_prompts/llm02_adversarial.txt +75 -0
  17. flintai/data/detector_prompts/llm02_fixed.txt +8 -0
  18. flintai/data/detector_prompts/llm05_adversarial.txt +63 -0
  19. flintai/data/detector_prompts/llm05_fixed.txt +16 -0
  20. flintai/data/detector_prompts/llm06_adversarial.txt +135 -0
  21. flintai/data/detector_prompts/llm06_fixed.txt +15 -0
  22. flintai/data/detector_prompts/llm07_adversarial.txt +185 -0
  23. flintai/data/detector_prompts/llm07_fixed.txt +14 -0
  24. flintai/data/detector_prompts/llm09_adversarial.txt +77 -0
  25. flintai/data/detector_prompts/llm09_fixed.txt +15 -0
  26. flintai/data/llm01_prompt_injection.csv +6708 -0
  27. flintai/data/llm02_sensitive_information.csv +1069 -0
  28. flintai/data/llm05_unsafe_output.csv +3882 -0
  29. flintai/data/llm06_excessive_agency.csv +778 -0
  30. flintai/data/llm07_system_prompt_leakage.csv +971 -0
  31. flintai/data/llm09_hallucination-goals.csv +32419 -0
  32. flintai/data/llm09_hallucination.csv +599 -0
  33. flintai/data/pii_leakage.csv +706 -0
  34. flintai/data/secret_leakage.csv +1380 -0
  35. flintai/eval/__init__.py +0 -0
  36. flintai/eval/common/converter_anthropic.py +151 -0
  37. flintai/eval/common/converter_genai.py +169 -0
  38. flintai/eval/common/converter_openai.py +165 -0
  39. flintai/eval/common/log.py +96 -0
  40. flintai/eval/common/reference.py +25 -0
  41. flintai/eval/common/schema.py +192 -0
  42. flintai/eval/common/utils.py +74 -0
  43. flintai/eval/core/__init__.py +0 -0
  44. flintai/eval/core/detectors/__init__.py +1 -0
  45. flintai/eval/core/detectors/detector.py +25 -0
  46. flintai/eval/core/detectors/detector_garak.py +80 -0
  47. flintai/eval/core/detectors/detector_model.py +76 -0
  48. flintai/eval/core/detectors/detector_pii.py +64 -0
  49. flintai/eval/core/detectors/detector_secret.py +82 -0
  50. flintai/eval/core/detectors/detector_topic_guard.py +70 -0
  51. flintai/eval/core/detectors/detector_toxicity.py +70 -0
  52. flintai/eval/core/eval/__init__.py +0 -0
  53. flintai/eval/core/eval/eval_creator.py +197 -0
  54. flintai/eval/core/eval/evaluation.py +100 -0
  55. flintai/eval/core/eval/evaluation_adversarial.py +523 -0
  56. flintai/eval/core/eval/evaluation_garak_module.py +52 -0
  57. flintai/eval/core/eval/evaluation_garak_probe.py +274 -0
  58. flintai/eval/core/eval/evaluation_message_list.py +44 -0
  59. flintai/eval/core/eval/evaluation_multi.py +207 -0
  60. flintai/eval/core/eval/evaluation_single.py +79 -0
  61. flintai/eval/core/eval/evaluation_single_prompt.py +58 -0
  62. flintai/eval/core/eval/evaluation_topic_guard.py +318 -0
  63. flintai/eval/core/eval/metric_conciseness.py +181 -0
  64. flintai/eval/core/eval/metric_factual_accuracy.py +294 -0
  65. flintai/eval/core/eval/metric_instruction_adherence.py +180 -0
  66. flintai/eval/core/eval/metric_tone.py +183 -0
  67. flintai/eval/core/eval/metric_toxicity.py +147 -0
  68. flintai/eval/core/eval/observer.py +136 -0
  69. flintai/eval/core/eval/probe_adversarial.py +14 -0
  70. flintai/eval/core/message/__init__.py +0 -0
  71. flintai/eval/core/message/message_collection.py +37 -0
  72. flintai/eval/core/message/message_collection_csv.py +46 -0
  73. flintai/eval/core/message/message_collection_garak.py +52 -0
  74. flintai/eval/core/message/message_collection_memory.py +36 -0
  75. flintai/eval/core/models/__init__.py +0 -0
  76. flintai/eval/core/models/generator_model.py +88 -0
  77. flintai/eval/core/models/model.py +111 -0
  78. flintai/eval/core/models/model_adk.py +158 -0
  79. flintai/eval/core/models/model_anthropic.py +51 -0
  80. flintai/eval/core/models/model_anthropic_agent.py +92 -0
  81. flintai/eval/core/models/model_gemini.py +107 -0
  82. flintai/eval/core/models/model_generic_http.py +117 -0
  83. flintai/eval/core/models/model_huggingface.py +61 -0
  84. flintai/eval/core/models/model_langserve.py +94 -0
  85. flintai/eval/core/models/model_litellm.py +38 -0
  86. flintai/eval/core/models/model_ollama.py +50 -0
  87. flintai/eval/core/models/model_openai.py +35 -0
  88. flintai/eval/core/models/model_openai_agent.py +80 -0
  89. flintai/eval/core/models/model_openai_compatible.py +57 -0
  90. flintai/eval/core/models/model_retry.py +134 -0
  91. flintai/eval/core/models/model_sync_wrapper.py +35 -0
  92. flintai/eval/db/__init__.py +0 -0
  93. flintai/eval/db/base/__init__.py +0 -0
  94. flintai/eval/db/base/detectors/__init__.py +1 -0
  95. flintai/eval/db/base/detectors/detector_helpers.py +78 -0
  96. flintai/eval/db/base/detectors/detector_repository.py +91 -0
  97. flintai/eval/db/base/detectors/detector_types.py +77 -0
  98. flintai/eval/db/base/eval/__init__.py +1 -0
  99. flintai/eval/db/base/eval/eval_helpers.py +227 -0
  100. flintai/eval/db/base/eval/eval_repository.py +115 -0
  101. flintai/eval/db/base/eval/eval_run.py +139 -0
  102. flintai/eval/db/base/eval/eval_types.py +129 -0
  103. flintai/eval/db/base/eval/model_eval_repository.py +78 -0
  104. flintai/eval/db/base/eval/model_eval_run_repository.py +70 -0
  105. flintai/eval/db/base/eval/model_eval_run_result_repository.py +40 -0
  106. flintai/eval/db/base/eval/model_eval_run_result_types.py +21 -0
  107. flintai/eval/db/base/eval/model_eval_run_types.py +84 -0
  108. flintai/eval/db/base/eval/model_eval_types.py +71 -0
  109. flintai/eval/db/base/message/__init__.py +0 -0
  110. flintai/eval/db/base/message/message_collection_helpers.py +61 -0
  111. flintai/eval/db/base/message/message_collection_repository.py +92 -0
  112. flintai/eval/db/base/message/message_collection_types.py +89 -0
  113. flintai/eval/db/base/models/__init__.py +0 -0
  114. flintai/eval/db/base/models/model_helpers.py +172 -0
  115. flintai/eval/db/base/models/model_repository.py +95 -0
  116. flintai/eval/db/base/models/model_types.py +156 -0
  117. flintai/eval/db/json/__init__.py +0 -0
  118. flintai/eval/db/json/repository_json.py +603 -0
  119. flintai/scan/__init__.py +0 -0
  120. flintai/scan/agent_scanner.py +797 -0
  121. flintai/scan/config/agent_opengrep_rules.yaml +533 -0
  122. flintai/scan/config/agent_taxonomy.json +481 -0
  123. flintai/scan/config/agentic_cvss_mapping.yaml +136 -0
  124. flintai/scan/config/compliance_mappings.json +110 -0
  125. flintai/scan/config/triage_prompt.txt +335 -0
  126. flintai/scan/constants.py +15 -0
  127. flintai/scan/file_filter.py +156 -0
  128. flintai/scan/llm_provider.py +197 -0
  129. flintai/scan/opengrep_resolver.py +12 -0
  130. flintai/scan/reasoner.py +552 -0
  131. flintai/scan/schema.py +337 -0
  132. flintai/scan/scorer.py +371 -0
  133. flintai/scan/secret_anonymizer.py +72 -0
  134. flintai/scan/static_scanner.py +542 -0
  135. flintai/scan/taxonomy.py +75 -0
  136. flintai/scan/tool_dispatcher.py +754 -0
  137. flintai/scan/trace_logger.py +118 -0
  138. flintai/scan/trace_logger_file.py +143 -0
  139. flintai/scan/trace_logger_log.py +100 -0
  140. flintai/scan/triage.py +496 -0
  141. flintai/schema.py +58 -0
  142. flintai_cli-1.0.0.dist-info/METADATA +442 -0
  143. flintai_cli-1.0.0.dist-info/RECORD +147 -0
  144. flintai_cli-1.0.0.dist-info/WHEEL +5 -0
  145. flintai_cli-1.0.0.dist-info/entry_points.txt +2 -0
  146. flintai_cli-1.0.0.dist-info/licenses/LICENSE +210 -0
  147. flintai_cli-1.0.0.dist-info/top_level.txt +1 -0
@@ -0,0 +1,197 @@
1
+ """
2
+ llm_provider.py — LLM model factory for the agent scanner.
3
+
4
+ Single entry point:
5
+
6
+ make_model(model_string)
7
+ Returns an ADK-compatible LiteLlm model object for all providers.
8
+ Used by both the agentic reasoner (ADK Runner) and single-pass
9
+ completions (triage) via complete_text().
10
+
11
+ complete_text(model, system_prompt, user_message, ...)
12
+ Runs a single-pass LLM completion using generate_content_async
13
+ on the model returned by make_model().
14
+
15
+ Configuration via AGENT_SCANNER_MODEL env var in 'provider:model' format
16
+ (e.g., 'google:gemini-2.5-flash', 'openai:gpt-5.4', 'anthropic:claude-sonnet-4-6').
17
+
18
+ API keys are read from standard env vars by the underlying frameworks:
19
+ GOOGLE_API_KEY, OPENAI_API_KEY, ANTHROPIC_API_KEY.
20
+ """
21
+
22
+ from __future__ import annotations
23
+
24
+ import asyncio
25
+ import concurrent.futures
26
+ import logging
27
+ import os
28
+ import re
29
+ import warnings
30
+ from typing import Any
31
+
32
+ from flintai.scan.schema import ADKModel
33
+
34
+ logger = logging.getLogger(__name__)
35
+
36
+ _REDACT_PATTERNS = re.compile(
37
+ r"("
38
+ r"Bearer\s+[A-Za-z0-9\-_.~+/]+=*"
39
+ r"|sk-[A-Za-z0-9]{10,}"
40
+ r"|key-[A-Za-z0-9]{10,}"
41
+ r"|pat-[A-Za-z0-9]{10,}"
42
+ r"|AIza[A-Za-z0-9_\-]{35,}"
43
+ r"|[A-Za-z0-9]{40,}"
44
+ r")",
45
+ re.IGNORECASE,
46
+ )
47
+
48
+
49
+ def _safe_error(exc: Exception) -> str:
50
+ """Return a log-safe string with token-like patterns redacted."""
51
+ return _REDACT_PATTERNS.sub("[REDACTED]", str(exc))
52
+
53
+
54
+ PROVIDER_GOOGLE = "google"
55
+ PROVIDER_LITELLM = "litellm"
56
+
57
+ DEFAULT_MODEL = "gemini-2.5-flash"
58
+
59
+ _PROVIDER_ALIASES = {"gemini": "google"}
60
+
61
+
62
+ def parse_model_string(model_string: str) -> tuple[str, str | None]:
63
+ """Parse 'provider:model' -> (provider, model)."""
64
+ if ":" in model_string:
65
+ provider, model = model_string.split(":", 1)
66
+ else:
67
+ provider, model = model_string, None
68
+ provider = provider.strip().lower()
69
+ provider = _PROVIDER_ALIASES.get(provider, provider)
70
+ return provider, model.strip() if model else None
71
+
72
+
73
+ def _resolve_model_string(model_string: str | None = None) -> tuple[str, str]:
74
+ """Resolve provider and model from argument or AGENT_SCANNER_MODEL env var."""
75
+ ms = (model_string or os.getenv("AGENT_SCANNER_MODEL", "")).strip()
76
+ if not ms:
77
+ return PROVIDER_GOOGLE, DEFAULT_MODEL
78
+ provider, model = parse_model_string(ms)
79
+ return provider, model or DEFAULT_MODEL
80
+
81
+
82
+ def make_model(model_string: str | None = None, temperature: float = 0.0) -> ADKModel:
83
+ """Return an ADK-compatible LiteLlm model for any provider.
84
+
85
+ Always returns a LiteLlm object so that all callers (agentic
86
+ reasoner, triage) use the same generate_content_async interface.
87
+ """
88
+ provider, model = _resolve_model_string(model_string)
89
+
90
+ LiteLlm = _import_litellm()
91
+ if provider == PROVIDER_GOOGLE:
92
+ with warnings.catch_warnings(record=True) as caught:
93
+ warnings.simplefilter("always")
94
+ result = LiteLlm(model=f"gemini/{model}", temperature=temperature)
95
+ for w in caught:
96
+ logger.debug("%s", w.message)
97
+ return result
98
+ if provider == PROVIDER_LITELLM:
99
+ return LiteLlm(model=model, temperature=temperature)
100
+ return LiteLlm(model=f"{provider}/{model}", temperature=temperature)
101
+
102
+
103
+ def get_model_name(model_string: str | None = None) -> str:
104
+ """Return the resolved model name for logging/metadata."""
105
+ _, model = _resolve_model_string(model_string)
106
+ return model
107
+
108
+
109
+ def _import_litellm() -> type[Any]:
110
+ """Import and return the LiteLlm class from google-adk or a minimal shim."""
111
+ try:
112
+ from google.adk.models.lite_llm import LiteLlm
113
+
114
+ return LiteLlm
115
+ except ImportError:
116
+ pass
117
+ try:
118
+ from google.adk.models import LiteLlm # type: ignore[no-redef]
119
+
120
+ return LiteLlm
121
+ except ImportError:
122
+ pass
123
+ try:
124
+ import litellm as _litellm # noqa: F401
125
+
126
+ class _LiteLlmShim:
127
+ def __init__(self, model: str, **kwargs):
128
+ self.model = model
129
+ self._kwargs = kwargs
130
+
131
+ def __repr__(self) -> str:
132
+ return f"LiteLlm(model={self.model!r})"
133
+
134
+ return _LiteLlmShim
135
+ except ImportError as e:
136
+ raise ImportError(
137
+ "google-adk or litellm is required for non-Google providers. "
138
+ "Run: pip install google-adk litellm"
139
+ ) from e
140
+
141
+
142
+ # ── Single-pass completion ───────────────────────────────────────────────────
143
+
144
+
145
+ def complete_text(
146
+ model: ADKModel,
147
+ system_prompt: str,
148
+ user_message: str,
149
+ max_tokens: int = 8000,
150
+ temperature: float = 0.0,
151
+ top_p: float = 1.0,
152
+ ) -> str | None:
153
+ """Run a single-pass LLM completion via model.generate_content_async."""
154
+ from google.adk.models.llm_request import LlmRequest # lazy: ADK optional
155
+ from google.genai import types as genai_types # lazy: ADK optional
156
+
157
+ model_name = getattr(model, "model", str(model))
158
+
159
+ config = genai_types.GenerateContentConfig(
160
+ system_instruction=system_prompt,
161
+ temperature=temperature,
162
+ top_p=top_p,
163
+ max_output_tokens=max_tokens,
164
+ )
165
+ contents = [
166
+ genai_types.Content(
167
+ role="user",
168
+ parts=[genai_types.Part.from_text(text=user_message)],
169
+ )
170
+ ]
171
+
172
+ async def _run() -> str | None:
173
+ request = LlmRequest(
174
+ model=model_name,
175
+ contents=contents,
176
+ config=config,
177
+ )
178
+ text = ""
179
+ async for resp in model.generate_content_async(request):
180
+ if resp.content and resp.content.parts:
181
+ for part in resp.content.parts:
182
+ if hasattr(part, "text") and part.text:
183
+ text += part.text
184
+ return text.strip() or None
185
+
186
+ try:
187
+ asyncio.get_running_loop()
188
+ with concurrent.futures.ThreadPoolExecutor() as pool:
189
+ return pool.submit(asyncio.run, _run()).result(timeout=120)
190
+ except RuntimeError:
191
+ pass
192
+
193
+ try:
194
+ return asyncio.run(_run())
195
+ except Exception as e:
196
+ logger.error("LLM call failed: %s", _safe_error(e))
197
+ return None
@@ -0,0 +1,12 @@
1
+ """
2
+ opengrep_resolver.py — Locate the OpenGrep binary.
3
+ """
4
+
5
+ import shutil
6
+
7
+
8
+ def find_opengrep_binary() -> str | None:
9
+ """
10
+ Returns the path to the binary, or None if not found.
11
+ """
12
+ return shutil.which("opengrep")